2024-12-02 14:18:53,926 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-02 14:18:53,940 main DEBUG Took 0.011697 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-02 14:18:53,940 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-02 14:18:53,941 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-02 14:18:53,942 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-02 14:18:53,943 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:18:53,952 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-02 14:18:53,966 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:18:53,968 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:18:53,968 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:18:53,969 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:18:53,970 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:18:53,970 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:18:53,971 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:18:53,971 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:18:53,972 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:18:53,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:18:53,974 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:18:53,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:18:53,975 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:18:53,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:18:53,976 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:18:53,976 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:18:53,977 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:18:53,977 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:18:53,978 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:18:53,978 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:18:53,978 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:18:53,979 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:18:53,979 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:18:53,979 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:18:53,980 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:18:53,980 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-02 14:18:53,982 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:18:53,983 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-02 14:18:53,986 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-02 14:18:53,986 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-02 14:18:53,988 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-02 14:18:53,988 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-02 14:18:53,998 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-02 14:18:54,001 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-02 14:18:54,003 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-02 14:18:54,004 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-02 14:18:54,004 main DEBUG createAppenders(={Console}) 2024-12-02 14:18:54,005 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-02 14:18:54,006 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-02 14:18:54,006 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-02 14:18:54,007 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-02 14:18:54,007 main DEBUG OutputStream closed 2024-12-02 14:18:54,007 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-02 14:18:54,008 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-02 14:18:54,008 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-02 14:18:54,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-02 14:18:54,095 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-02 14:18:54,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-02 14:18:54,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-02 14:18:54,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-02 14:18:54,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-02 14:18:54,099 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-02 14:18:54,099 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-02 14:18:54,099 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-02 14:18:54,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-02 14:18:54,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-02 14:18:54,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-02 14:18:54,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-02 14:18:54,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-02 14:18:54,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-02 14:18:54,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-02 14:18:54,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-02 14:18:54,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-02 14:18:54,106 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02 14:18:54,106 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-02 14:18:54,107 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-02 14:18:54,108 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-02T14:18:54,389 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338 2024-12-02 14:18:54,393 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-02 14:18:54,393 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02T14:18:54,404 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-02T14:18:54,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T14:18:54,447 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/cluster_c8d22f2e-58bf-6f09-e5bc-40f0041042bb, deleteOnExit=true 2024-12-02T14:18:54,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T14:18:54,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/test.cache.data in system properties and HBase conf 2024-12-02T14:18:54,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T14:18:54,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/hadoop.log.dir in system properties and HBase conf 2024-12-02T14:18:54,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T14:18:54,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T14:18:54,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T14:18:54,559 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-02T14:18:54,665 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T14:18:54,669 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T14:18:54,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T14:18:54,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T14:18:54,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T14:18:54,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T14:18:54,672 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T14:18:54,672 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T14:18:54,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T14:18:54,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T14:18:54,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/nfs.dump.dir in system properties and HBase conf 2024-12-02T14:18:54,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/java.io.tmpdir in system properties and HBase conf 2024-12-02T14:18:54,675 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T14:18:54,675 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T14:18:54,675 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T14:18:55,432 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-02T14:18:55,508 INFO [Time-limited test {}] log.Log(170): Logging initialized @2310ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-02T14:18:55,574 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T14:18:55,629 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T14:18:55,647 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T14:18:55,647 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T14:18:55,648 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T14:18:55,659 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T14:18:55,661 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/hadoop.log.dir/,AVAILABLE} 2024-12-02T14:18:55,662 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T14:18:55,874 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/java.io.tmpdir/jetty-localhost-38401-hadoop-hdfs-3_4_1-tests_jar-_-any-17499396811348457545/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T14:18:55,882 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:38401} 2024-12-02T14:18:55,883 INFO [Time-limited test {}] server.Server(415): Started @2685ms 2024-12-02T14:18:56,275 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T14:18:56,283 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T14:18:56,284 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T14:18:56,284 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T14:18:56,284 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T14:18:56,286 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6aa6ac55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/hadoop.log.dir/,AVAILABLE} 2024-12-02T14:18:56,286 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58f8c8ee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T14:18:56,394 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d1d0948{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/java.io.tmpdir/jetty-localhost-39195-hadoop-hdfs-3_4_1-tests_jar-_-any-13747891389486380097/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T14:18:56,395 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@49b49063{HTTP/1.1, (http/1.1)}{localhost:39195} 2024-12-02T14:18:56,395 INFO [Time-limited test {}] server.Server(415): Started @3198ms 2024-12-02T14:18:56,442 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T14:18:56,842 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/cluster_c8d22f2e-58bf-6f09-e5bc-40f0041042bb/data/data1/current/BP-2028019360-172.17.0.2-1733149135234/current, will proceed with Du for space computation calculation, 2024-12-02T14:18:56,842 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/cluster_c8d22f2e-58bf-6f09-e5bc-40f0041042bb/data/data2/current/BP-2028019360-172.17.0.2-1733149135234/current, will proceed with Du for space computation calculation, 2024-12-02T14:18:56,906 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T14:18:56,962 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x200afa2157a511e9 with lease ID 0x667e710e24c4201e: Processing first storage report for DS-5415e89b-8d1c-4521-8651-a8378cbbb10c from datanode DatanodeRegistration(127.0.0.1:41597, datanodeUuid=c66cc415-bb0e-464a-9540-4f25224042e8, infoPort=33207, infoSecurePort=0, ipcPort=34511, storageInfo=lv=-57;cid=testClusterID;nsid=311464454;c=1733149135234) 2024-12-02T14:18:56,963 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x200afa2157a511e9 with lease ID 0x667e710e24c4201e: from storage DS-5415e89b-8d1c-4521-8651-a8378cbbb10c node DatanodeRegistration(127.0.0.1:41597, datanodeUuid=c66cc415-bb0e-464a-9540-4f25224042e8, infoPort=33207, infoSecurePort=0, ipcPort=34511, storageInfo=lv=-57;cid=testClusterID;nsid=311464454;c=1733149135234), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T14:18:56,964 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x200afa2157a511e9 with lease ID 0x667e710e24c4201e: Processing first storage report for DS-a02d2000-57b2-45fe-a937-aa509b78eb29 from datanode DatanodeRegistration(127.0.0.1:41597, datanodeUuid=c66cc415-bb0e-464a-9540-4f25224042e8, infoPort=33207, infoSecurePort=0, ipcPort=34511, storageInfo=lv=-57;cid=testClusterID;nsid=311464454;c=1733149135234) 2024-12-02T14:18:56,964 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x200afa2157a511e9 with lease ID 0x667e710e24c4201e: from storage DS-a02d2000-57b2-45fe-a937-aa509b78eb29 node DatanodeRegistration(127.0.0.1:41597, datanodeUuid=c66cc415-bb0e-464a-9540-4f25224042e8, infoPort=33207, infoSecurePort=0, ipcPort=34511, storageInfo=lv=-57;cid=testClusterID;nsid=311464454;c=1733149135234), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T14:18:57,051 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338 2024-12-02T14:18:57,119 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/cluster_c8d22f2e-58bf-6f09-e5bc-40f0041042bb/zookeeper_0, clientPort=51703, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/cluster_c8d22f2e-58bf-6f09-e5bc-40f0041042bb/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/cluster_c8d22f2e-58bf-6f09-e5bc-40f0041042bb/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T14:18:57,130 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51703 2024-12-02T14:18:57,140 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T14:18:57,143 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T14:18:57,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741825_1001 (size=7) 2024-12-02T14:18:57,794 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919 with version=8 2024-12-02T14:18:57,794 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/hbase-staging 2024-12-02T14:18:57,888 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-02T14:18:58,098 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a3a61c9ba14f:0 server-side Connection retries=45 2024-12-02T14:18:58,107 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T14:18:58,107 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T14:18:58,111 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T14:18:58,111 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T14:18:58,112 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T14:18:58,225 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T14:18:58,277 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-02T14:18:58,285 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-02T14:18:58,289 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T14:18:58,311 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 97288 (auto-detected) 2024-12-02T14:18:58,312 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-02T14:18:58,330 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35877 2024-12-02T14:18:58,349 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35877 connecting to ZooKeeper ensemble=127.0.0.1:51703 2024-12-02T14:18:58,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:358770x0, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T14:18:58,382 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35877-0x1009b4c753c0000 connected 2024-12-02T14:18:58,428 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T14:18:58,431 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T14:18:58,441 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T14:18:58,447 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919, hbase.cluster.distributed=false 2024-12-02T14:18:58,472 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T14:18:58,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35877 2024-12-02T14:18:58,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35877 2024-12-02T14:18:58,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35877 2024-12-02T14:18:58,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35877 2024-12-02T14:18:58,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35877 2024-12-02T14:18:58,573 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a3a61c9ba14f:0 server-side Connection retries=45 2024-12-02T14:18:58,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T14:18:58,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T14:18:58,575 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T14:18:58,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T14:18:58,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T14:18:58,578 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T14:18:58,580 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T14:18:58,580 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46793 2024-12-02T14:18:58,582 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46793 connecting to ZooKeeper ensemble=127.0.0.1:51703 2024-12-02T14:18:58,583 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T14:18:58,586 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T14:18:58,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467930x0, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T14:18:58,593 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:467930x0, quorum=127.0.0.1:51703, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T14:18:58,593 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46793-0x1009b4c753c0001 connected 2024-12-02T14:18:58,596 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T14:18:58,603 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T14:18:58,606 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T14:18:58,610 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T14:18:58,611 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46793 2024-12-02T14:18:58,613 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46793 2024-12-02T14:18:58,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46793 2024-12-02T14:18:58,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46793 2024-12-02T14:18:58,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46793 2024-12-02T14:18:58,643 DEBUG [M:0;a3a61c9ba14f:35877 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a3a61c9ba14f:35877 2024-12-02T14:18:58,644 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a3a61c9ba14f,35877,1733149137941 2024-12-02T14:18:58,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T14:18:58,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T14:18:58,653 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a3a61c9ba14f,35877,1733149137941 2024-12-02T14:18:58,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T14:18:58,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:18:58,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:18:58,678 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T14:18:58,679 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a3a61c9ba14f,35877,1733149137941 from backup master directory 2024-12-02T14:18:58,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T14:18:58,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a3a61c9ba14f,35877,1733149137941 2024-12-02T14:18:58,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T14:18:58,682 WARN [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T14:18:58,682 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a3a61c9ba14f,35877,1733149137941 2024-12-02T14:18:58,684 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-02T14:18:58,686 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-02T14:18:58,734 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/hbase.id] with ID: 26e152f2-8fae-48f3-8f83-0754b5a7150f 2024-12-02T14:18:58,734 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/.tmp/hbase.id 2024-12-02T14:18:58,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741826_1002 (size=42) 2024-12-02T14:18:58,744 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/.tmp/hbase.id]:[hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/hbase.id] 2024-12-02T14:18:58,786 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T14:18:58,790 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T14:18:58,807 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-02T14:18:58,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:18:58,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:18:58,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741827_1003 (size=196) 2024-12-02T14:18:58,836 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T14:18:58,838 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T14:18:58,852 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:18:58,857 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T14:18:58,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741828_1004 (size=1189) 2024-12-02T14:18:59,301 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store 2024-12-02T14:18:59,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741829_1005 (size=34) 2024-12-02T14:18:59,321 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-02T14:18:59,323 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:18:59,324 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T14:18:59,324 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T14:18:59,325 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T14:18:59,326 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T14:18:59,326 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T14:18:59,326 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T14:18:59,327 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733149139324Disabling compacts and flushes for region at 1733149139324Disabling writes for close at 1733149139326 (+2 ms)Writing region close event to WAL at 1733149139326Closed at 1733149139326 2024-12-02T14:18:59,329 WARN [master/a3a61c9ba14f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/.initializing 2024-12-02T14:18:59,329 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/WALs/a3a61c9ba14f,35877,1733149137941 2024-12-02T14:18:59,337 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T14:18:59,351 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a3a61c9ba14f%2C35877%2C1733149137941, suffix=, logDir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/WALs/a3a61c9ba14f,35877,1733149137941, archiveDir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/oldWALs, maxLogs=10 2024-12-02T14:18:59,370 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/WALs/a3a61c9ba14f,35877,1733149137941/a3a61c9ba14f%2C35877%2C1733149137941.1733149139355, exclude list is [], retry=0 2024-12-02T14:18:59,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41597,DS-5415e89b-8d1c-4521-8651-a8378cbbb10c,DISK] 2024-12-02T14:18:59,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-02T14:18:59,423 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/WALs/a3a61c9ba14f,35877,1733149137941/a3a61c9ba14f%2C35877%2C1733149137941.1733149139355 2024-12-02T14:18:59,424 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33207:33207)] 2024-12-02T14:18:59,425 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T14:18:59,425 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:18:59,429 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:18:59,430 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:18:59,462 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:18:59,483 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T14:18:59,487 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:18:59,489 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:18:59,489 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:18:59,492 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T14:18:59,493 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:18:59,494 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:18:59,494 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:18:59,496 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T14:18:59,496 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:18:59,497 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:18:59,497 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:18:59,499 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T14:18:59,500 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:18:59,500 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:18:59,501 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:18:59,504 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:18:59,506 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:18:59,511 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:18:59,512 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:18:59,515 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T14:18:59,520 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:18:59,524 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T14:18:59,526 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61319004, jitterRate=-0.08627563714981079}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T14:18:59,531 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733149139442Initializing all the Stores at 1733149139444 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733149139444Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149139445 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149139445Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149139445Cleaning up temporary data from old regions at 1733149139512 (+67 ms)Region opened successfully at 1733149139531 (+19 ms) 2024-12-02T14:18:59,532 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T14:18:59,561 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@307ec53d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a3a61c9ba14f/172.17.0.2:0 2024-12-02T14:18:59,592 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T14:18:59,605 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T14:18:59,605 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T14:18:59,608 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T14:18:59,610 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-02T14:18:59,616 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-02T14:18:59,616 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T14:18:59,647 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T14:18:59,659 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T14:18:59,661 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T14:18:59,665 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T14:18:59,667 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T14:18:59,668 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T14:18:59,672 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T14:18:59,676 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T14:18:59,678 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T14:18:59,680 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T14:18:59,681 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T14:18:59,703 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T14:18:59,704 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T14:18:59,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T14:18:59,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:18:59,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T14:18:59,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:18:59,713 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a3a61c9ba14f,35877,1733149137941, sessionid=0x1009b4c753c0000, setting cluster-up flag (Was=false) 2024-12-02T14:18:59,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:18:59,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:18:59,730 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T14:18:59,732 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a3a61c9ba14f,35877,1733149137941 2024-12-02T14:18:59,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:18:59,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:18:59,743 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T14:18:59,744 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a3a61c9ba14f,35877,1733149137941 2024-12-02T14:18:59,752 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T14:18:59,819 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T14:18:59,828 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T14:18:59,829 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(746): ClusterId : 26e152f2-8fae-48f3-8f83-0754b5a7150f 2024-12-02T14:18:59,832 DEBUG [RS:0;a3a61c9ba14f:46793 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T14:18:59,834 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T14:18:59,836 DEBUG [RS:0;a3a61c9ba14f:46793 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T14:18:59,836 DEBUG [RS:0;a3a61c9ba14f:46793 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T14:18:59,841 DEBUG [RS:0;a3a61c9ba14f:46793 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T14:18:59,841 DEBUG [RS:0;a3a61c9ba14f:46793 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7320d5fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a3a61c9ba14f/172.17.0.2:0 2024-12-02T14:18:59,840 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a3a61c9ba14f,35877,1733149137941 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T14:18:59,847 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a3a61c9ba14f:0, corePoolSize=5, maxPoolSize=5 2024-12-02T14:18:59,848 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a3a61c9ba14f:0, corePoolSize=5, maxPoolSize=5 2024-12-02T14:18:59,848 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a3a61c9ba14f:0, corePoolSize=5, maxPoolSize=5 2024-12-02T14:18:59,848 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a3a61c9ba14f:0, corePoolSize=5, maxPoolSize=5 2024-12-02T14:18:59,849 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a3a61c9ba14f:0, corePoolSize=10, maxPoolSize=10 2024-12-02T14:18:59,849 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:18:59,849 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a3a61c9ba14f:0, corePoolSize=2, maxPoolSize=2 2024-12-02T14:18:59,849 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:18:59,859 DEBUG [RS:0;a3a61c9ba14f:46793 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a3a61c9ba14f:46793 2024-12-02T14:18:59,859 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733149169859 2024-12-02T14:18:59,861 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T14:18:59,861 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T14:18:59,861 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T14:18:59,862 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T14:18:59,862 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T14:18:59,862 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T14:18:59,862 DEBUG [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T14:18:59,865 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T14:18:59,865 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(2659): reportForDuty to master=a3a61c9ba14f,35877,1733149137941 with port=46793, startcode=1733149138543 2024-12-02T14:18:59,865 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T14:18:59,866 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T14:18:59,866 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T14:18:59,867 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:18:59,868 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T14:18:59,869 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T14:18:59,872 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T14:18:59,873 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T14:18:59,874 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T14:18:59,876 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T14:18:59,876 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T14:18:59,877 DEBUG [RS:0;a3a61c9ba14f:46793 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T14:18:59,878 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a3a61c9ba14f:0:becomeActiveMaster-HFileCleaner.large.0-1733149139877,5,FailOnTimeoutGroup] 2024-12-02T14:18:59,881 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a3a61c9ba14f:0:becomeActiveMaster-HFileCleaner.small.0-1733149139879,5,FailOnTimeoutGroup] 2024-12-02T14:18:59,881 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T14:18:59,881 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T14:18:59,883 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T14:18:59,883 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T14:18:59,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741831_1007 (size=1321) 2024-12-02T14:18:59,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44915, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T14:18:59,919 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35877 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a3a61c9ba14f,46793,1733149138543 2024-12-02T14:18:59,921 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35877 {}] master.ServerManager(517): Registering regionserver=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:18:59,937 DEBUG [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919 2024-12-02T14:18:59,937 DEBUG [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38859 2024-12-02T14:18:59,937 DEBUG [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T14:18:59,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T14:18:59,941 DEBUG [RS:0;a3a61c9ba14f:46793 {}] zookeeper.ZKUtil(111): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a3a61c9ba14f,46793,1733149138543 2024-12-02T14:18:59,941 WARN [RS:0;a3a61c9ba14f:46793 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T14:18:59,941 INFO [RS:0;a3a61c9ba14f:46793 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T14:18:59,941 DEBUG [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/WALs/a3a61c9ba14f,46793,1733149138543 2024-12-02T14:18:59,947 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a3a61c9ba14f,46793,1733149138543] 2024-12-02T14:18:59,973 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T14:18:59,989 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T14:18:59,995 INFO [RS:0;a3a61c9ba14f:46793 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T14:18:59,995 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T14:18:59,997 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T14:19:00,003 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T14:19:00,005 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T14:19:00,005 DEBUG [RS:0;a3a61c9ba14f:46793 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:19:00,005 DEBUG [RS:0;a3a61c9ba14f:46793 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:19:00,005 DEBUG [RS:0;a3a61c9ba14f:46793 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:19:00,005 DEBUG [RS:0;a3a61c9ba14f:46793 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:19:00,006 DEBUG [RS:0;a3a61c9ba14f:46793 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:19:00,006 DEBUG [RS:0;a3a61c9ba14f:46793 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a3a61c9ba14f:0, corePoolSize=2, maxPoolSize=2 2024-12-02T14:19:00,006 DEBUG [RS:0;a3a61c9ba14f:46793 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:19:00,006 DEBUG [RS:0;a3a61c9ba14f:46793 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:19:00,006 DEBUG [RS:0;a3a61c9ba14f:46793 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:19:00,006 DEBUG [RS:0;a3a61c9ba14f:46793 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:19:00,006 DEBUG [RS:0;a3a61c9ba14f:46793 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:19:00,006 DEBUG [RS:0;a3a61c9ba14f:46793 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:19:00,007 DEBUG [RS:0;a3a61c9ba14f:46793 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a3a61c9ba14f:0, corePoolSize=3, maxPoolSize=3 2024-12-02T14:19:00,007 DEBUG [RS:0;a3a61c9ba14f:46793 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0, corePoolSize=3, maxPoolSize=3 2024-12-02T14:19:00,009 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T14:19:00,009 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T14:19:00,009 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T14:19:00,009 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T14:19:00,009 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T14:19:00,010 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3a61c9ba14f,46793,1733149138543-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T14:19:00,034 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T14:19:00,036 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3a61c9ba14f,46793,1733149138543-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T14:19:00,036 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T14:19:00,037 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.Replication(171): a3a61c9ba14f,46793,1733149138543 started 2024-12-02T14:19:00,060 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T14:19:00,060 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1482): Serving as a3a61c9ba14f,46793,1733149138543, RpcServer on a3a61c9ba14f/172.17.0.2:46793, sessionid=0x1009b4c753c0001 2024-12-02T14:19:00,061 DEBUG [RS:0;a3a61c9ba14f:46793 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T14:19:00,061 DEBUG [RS:0;a3a61c9ba14f:46793 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:00,062 DEBUG [RS:0;a3a61c9ba14f:46793 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3a61c9ba14f,46793,1733149138543' 2024-12-02T14:19:00,062 DEBUG [RS:0;a3a61c9ba14f:46793 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T14:19:00,063 DEBUG [RS:0;a3a61c9ba14f:46793 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T14:19:00,064 DEBUG [RS:0;a3a61c9ba14f:46793 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T14:19:00,064 DEBUG [RS:0;a3a61c9ba14f:46793 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T14:19:00,064 DEBUG [RS:0;a3a61c9ba14f:46793 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:00,064 DEBUG [RS:0;a3a61c9ba14f:46793 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3a61c9ba14f,46793,1733149138543' 2024-12-02T14:19:00,064 DEBUG [RS:0;a3a61c9ba14f:46793 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T14:19:00,065 DEBUG [RS:0;a3a61c9ba14f:46793 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T14:19:00,065 DEBUG [RS:0;a3a61c9ba14f:46793 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T14:19:00,066 INFO [RS:0;a3a61c9ba14f:46793 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T14:19:00,066 INFO [RS:0;a3a61c9ba14f:46793 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T14:19:00,171 INFO [RS:0;a3a61c9ba14f:46793 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T14:19:00,174 INFO [RS:0;a3a61c9ba14f:46793 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a3a61c9ba14f%2C46793%2C1733149138543, suffix=, logDir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/WALs/a3a61c9ba14f,46793,1733149138543, archiveDir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/oldWALs, maxLogs=32 2024-12-02T14:19:00,193 DEBUG [RS:0;a3a61c9ba14f:46793 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/WALs/a3a61c9ba14f,46793,1733149138543/a3a61c9ba14f%2C46793%2C1733149138543.1733149140177, exclude list is [], retry=0 2024-12-02T14:19:00,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41597,DS-5415e89b-8d1c-4521-8651-a8378cbbb10c,DISK] 2024-12-02T14:19:00,201 INFO [RS:0;a3a61c9ba14f:46793 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/WALs/a3a61c9ba14f,46793,1733149138543/a3a61c9ba14f%2C46793%2C1733149138543.1733149140177 2024-12-02T14:19:00,201 DEBUG [RS:0;a3a61c9ba14f:46793 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33207:33207)] 2024-12-02T14:19:00,292 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T14:19:00,293 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919 2024-12-02T14:19:00,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741833_1009 (size=32) 2024-12-02T14:19:00,705 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:19:00,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T14:19:00,710 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T14:19:00,710 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:00,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:19:00,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T14:19:00,714 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T14:19:00,714 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:00,715 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:19:00,715 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T14:19:00,718 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T14:19:00,718 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:00,719 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:19:00,719 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T14:19:00,722 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T14:19:00,722 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:00,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:19:00,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T14:19:00,725 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740 2024-12-02T14:19:00,725 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740 2024-12-02T14:19:00,728 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T14:19:00,728 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T14:19:00,729 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T14:19:00,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T14:19:00,736 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T14:19:00,737 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63120427, jitterRate=-0.059432342648506165}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T14:19:00,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733149140705Initializing all the Stores at 1733149140707 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733149140707Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733149140707Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149140707Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733149140707Cleaning up temporary data from old regions at 1733149140728 (+21 ms)Region opened successfully at 1733149140739 (+11 ms) 2024-12-02T14:19:00,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T14:19:00,739 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T14:19:00,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T14:19:00,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T14:19:00,740 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T14:19:00,741 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T14:19:00,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733149140739Disabling compacts and flushes for region at 1733149140739Disabling writes for close at 1733149140739Writing region close event to WAL at 1733149140741 (+2 ms)Closed at 1733149140741 2024-12-02T14:19:00,744 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T14:19:00,745 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T14:19:00,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T14:19:00,761 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T14:19:00,763 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T14:19:00,915 DEBUG [a3a61c9ba14f:35877 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-02T14:19:00,925 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:00,931 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a3a61c9ba14f,46793,1733149138543, state=OPENING 2024-12-02T14:19:00,935 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T14:19:00,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:19:00,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:19:00,937 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T14:19:00,937 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T14:19:00,938 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T14:19:00,940 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:19:01,110 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T14:19:01,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60409, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T14:19:01,123 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T14:19:01,123 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T14:19:01,123 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T14:19:01,127 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a3a61c9ba14f%2C46793%2C1733149138543.meta, suffix=.meta, logDir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/WALs/a3a61c9ba14f,46793,1733149138543, archiveDir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/oldWALs, maxLogs=32 2024-12-02T14:19:01,146 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/WALs/a3a61c9ba14f,46793,1733149138543/a3a61c9ba14f%2C46793%2C1733149138543.meta.1733149141129.meta, exclude list is [], retry=0 2024-12-02T14:19:01,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41597,DS-5415e89b-8d1c-4521-8651-a8378cbbb10c,DISK] 2024-12-02T14:19:01,152 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/WALs/a3a61c9ba14f,46793,1733149138543/a3a61c9ba14f%2C46793%2C1733149138543.meta.1733149141129.meta 2024-12-02T14:19:01,153 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33207:33207)] 2024-12-02T14:19:01,153 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T14:19:01,155 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T14:19:01,157 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T14:19:01,162 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T14:19:01,165 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T14:19:01,166 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:19:01,166 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T14:19:01,166 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T14:19:01,170 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T14:19:01,171 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T14:19:01,171 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:01,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:19:01,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T14:19:01,174 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T14:19:01,174 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:01,175 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:19:01,175 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T14:19:01,176 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T14:19:01,176 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:01,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:19:01,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T14:19:01,179 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T14:19:01,179 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:01,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:19:01,180 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T14:19:01,181 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740 2024-12-02T14:19:01,184 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740 2024-12-02T14:19:01,187 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T14:19:01,187 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T14:19:01,188 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T14:19:01,191 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T14:19:01,193 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75030788, jitterRate=0.1180458664894104}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T14:19:01,193 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T14:19:01,195 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733149141167Writing region info on filesystem at 1733149141167Initializing all the Stores at 1733149141169 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733149141169Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733149141169Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149141169Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733149141169Cleaning up temporary data from old regions at 1733149141187 (+18 ms)Running coprocessor post-open hooks at 1733149141194 (+7 ms)Region opened successfully at 1733149141195 (+1 ms) 2024-12-02T14:19:01,204 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733149141103 2024-12-02T14:19:01,218 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T14:19:01,219 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T14:19:01,221 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:01,224 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a3a61c9ba14f,46793,1733149138543, state=OPEN 2024-12-02T14:19:01,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T14:19:01,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T14:19:01,228 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T14:19:01,228 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T14:19:01,228 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:01,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T14:19:01,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a3a61c9ba14f,46793,1733149138543 in 288 msec 2024-12-02T14:19:01,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T14:19:01,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 484 msec 2024-12-02T14:19:01,246 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T14:19:01,246 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T14:19:01,270 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:01,272 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:01,296 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:01,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53261, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:01,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5400 sec 2024-12-02T14:19:01,321 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733149141321, completionTime=-1 2024-12-02T14:19:01,324 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-02T14:19:01,324 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T14:19:01,372 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-02T14:19:01,372 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733149201372 2024-12-02T14:19:01,372 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733149261372 2024-12-02T14:19:01,372 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 48 msec 2024-12-02T14:19:01,378 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3a61c9ba14f,35877,1733149137941-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T14:19:01,378 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3a61c9ba14f,35877,1733149137941-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T14:19:01,378 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3a61c9ba14f,35877,1733149137941-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T14:19:01,381 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a3a61c9ba14f:35877, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T14:19:01,381 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T14:19:01,386 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T14:19:01,390 DEBUG [master/a3a61c9ba14f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T14:19:01,414 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.731sec 2024-12-02T14:19:01,415 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T14:19:01,416 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T14:19:01,417 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T14:19:01,417 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T14:19:01,418 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T14:19:01,418 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3a61c9ba14f,35877,1733149137941-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T14:19:01,419 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3a61c9ba14f,35877,1733149137941-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T14:19:01,426 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T14:19:01,427 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T14:19:01,428 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3a61c9ba14f,35877,1733149137941-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T14:19:01,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4799df63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:01,439 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-02T14:19:01,439 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-02T14:19:01,442 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:01,444 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:01,452 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:01,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:01,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4843af8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:01,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:01,461 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:01,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f66000e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:01,465 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:01,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:01,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:01,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a3a61c9ba14f,35877,1733149137941 2024-12-02T14:19:01,505 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=321, ProcessCount=11, AvailableMemoryMB=5669 2024-12-02T14:19:01,513 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T14:19:01,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a3a61c9ba14f,35877,1733149137941 2024-12-02T14:19:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27423f7d 2024-12-02T14:19:01,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T14:19:01,531 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T14:19:01,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T14:19:01,538 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35877 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T14:19:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T14:19:01,551 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T14:19:01,553 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35877 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-02T14:19:01,554 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:01,556 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T14:19:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T14:19:01,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741835_1011 (size=1061) 2024-12-02T14:19:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T14:19:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T14:19:01,976 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919 2024-12-02T14:19:01,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741836_1012 (size=53) 2024-12-02T14:19:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T14:19:02,397 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:19:02,397 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4eaa960f4ddde5e2082f5ac5eba8cec8, disabling compactions & flushes 2024-12-02T14:19:02,397 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:02,397 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:02,397 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. after waiting 0 ms 2024-12-02T14:19:02,397 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:02,397 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:02,397 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: Waiting for close lock at 1733149142397Disabling compacts and flushes for region at 1733149142397Disabling writes for close at 1733149142397Writing region close event to WAL at 1733149142397Closed at 1733149142397 2024-12-02T14:19:02,399 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T14:19:02,404 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733149142400"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733149142400"}]},"ts":"1733149142400"} 2024-12-02T14:19:02,409 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T14:19:02,411 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T14:19:02,414 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149142411"}]},"ts":"1733149142411"} 2024-12-02T14:19:02,418 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T14:19:02,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4eaa960f4ddde5e2082f5ac5eba8cec8, ASSIGN}] 2024-12-02T14:19:02,422 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4eaa960f4ddde5e2082f5ac5eba8cec8, ASSIGN 2024-12-02T14:19:02,424 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4eaa960f4ddde5e2082f5ac5eba8cec8, ASSIGN; state=OFFLINE, location=a3a61c9ba14f,46793,1733149138543; forceNewPlan=false, retain=false 2024-12-02T14:19:02,575 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4eaa960f4ddde5e2082f5ac5eba8cec8, regionState=OPENING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:02,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4eaa960f4ddde5e2082f5ac5eba8cec8, ASSIGN because future has completed 2024-12-02T14:19:02,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:19:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T14:19:02,740 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:02,741 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} 2024-12-02T14:19:02,741 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:02,741 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:19:02,742 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:02,742 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:02,744 INFO [StoreOpener-4eaa960f4ddde5e2082f5ac5eba8cec8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:02,747 INFO [StoreOpener-4eaa960f4ddde5e2082f5ac5eba8cec8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:19:02,747 INFO [StoreOpener-4eaa960f4ddde5e2082f5ac5eba8cec8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4eaa960f4ddde5e2082f5ac5eba8cec8 columnFamilyName A 2024-12-02T14:19:02,748 DEBUG [StoreOpener-4eaa960f4ddde5e2082f5ac5eba8cec8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:02,749 INFO [StoreOpener-4eaa960f4ddde5e2082f5ac5eba8cec8-1 {}] regionserver.HStore(327): Store=4eaa960f4ddde5e2082f5ac5eba8cec8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:19:02,749 INFO [StoreOpener-4eaa960f4ddde5e2082f5ac5eba8cec8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:02,751 INFO [StoreOpener-4eaa960f4ddde5e2082f5ac5eba8cec8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:19:02,751 INFO [StoreOpener-4eaa960f4ddde5e2082f5ac5eba8cec8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4eaa960f4ddde5e2082f5ac5eba8cec8 columnFamilyName B 2024-12-02T14:19:02,751 DEBUG [StoreOpener-4eaa960f4ddde5e2082f5ac5eba8cec8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:02,752 INFO [StoreOpener-4eaa960f4ddde5e2082f5ac5eba8cec8-1 {}] regionserver.HStore(327): Store=4eaa960f4ddde5e2082f5ac5eba8cec8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:19:02,752 INFO [StoreOpener-4eaa960f4ddde5e2082f5ac5eba8cec8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:02,754 INFO [StoreOpener-4eaa960f4ddde5e2082f5ac5eba8cec8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:19:02,755 INFO [StoreOpener-4eaa960f4ddde5e2082f5ac5eba8cec8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4eaa960f4ddde5e2082f5ac5eba8cec8 columnFamilyName C 2024-12-02T14:19:02,755 DEBUG [StoreOpener-4eaa960f4ddde5e2082f5ac5eba8cec8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:02,756 INFO [StoreOpener-4eaa960f4ddde5e2082f5ac5eba8cec8-1 {}] regionserver.HStore(327): Store=4eaa960f4ddde5e2082f5ac5eba8cec8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:19:02,756 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:02,756 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:02,758 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:02,759 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:02,761 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:02,761 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:02,762 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T14:19:02,764 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:02,768 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T14:19:02,769 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4eaa960f4ddde5e2082f5ac5eba8cec8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67754357, jitterRate=0.009618595242500305}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T14:19:02,769 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:02,770 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: Running coprocessor pre-open hook at 1733149142742Writing region info on filesystem at 1733149142742Initializing all the Stores at 1733149142743 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149142743Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149142744 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149142744Cleaning up temporary data from old regions at 1733149142761 (+17 ms)Running coprocessor post-open hooks at 1733149142769 (+8 ms)Region opened successfully at 1733149142770 (+1 ms) 2024-12-02T14:19:02,772 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., pid=6, masterSystemTime=1733149142734 2024-12-02T14:19:02,776 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:02,776 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:02,777 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4eaa960f4ddde5e2082f5ac5eba8cec8, regionState=OPEN, openSeqNum=2, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:02,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:19:02,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-02T14:19:02,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 in 204 msec 2024-12-02T14:19:02,794 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-02T14:19:02,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4eaa960f4ddde5e2082f5ac5eba8cec8, ASSIGN in 369 msec 2024-12-02T14:19:02,796 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T14:19:02,797 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149142796"}]},"ts":"1733149142796"} 2024-12-02T14:19:02,800 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T14:19:02,802 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T14:19:02,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2610 sec 2024-12-02T14:19:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T14:19:03,710 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:03,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75877970, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:03,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:03,716 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d2f00be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:03,719 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c7d93ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,720 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:03,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:03,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:03,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:03,726 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T14:19:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is a3a61c9ba14f,35877,1733149137941 2024-12-02T14:19:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7aa46f1e 2024-12-02T14:19:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T14:19:03,733 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T14:19:03,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a0c60c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,740 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:03,740 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:03,742 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa64271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:03,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:03,751 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:03,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7685e9fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,753 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:03,753 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:03,757 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0e6697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:03,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:03,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:03,761 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec6ad86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e3eb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:03,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:03,770 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:03,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540312d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:03,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:03,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:03,774 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:03,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74cae7c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4d8404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,777 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:03,777 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:03,781 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7747469c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:03,786 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:03,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34c21631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382062a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:03,790 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:03,791 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:03,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:03,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d835570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:03,794 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8d6bd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53cb06a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,797 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:03,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:03,798 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:03,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc03052, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:03,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:03,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:03,801 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:03,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267419b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ef992a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:03,806 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:03,807 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62d09ecd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:03,809 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e1495ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b28440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:03,813 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:03,814 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fe58dfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:03,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:03,817 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a18c602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64696a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,818 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:03,819 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:03,819 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@159e37c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:03,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:03,821 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ad5b9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:03,827 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:03,827 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:03,828 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:03,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:03,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:03,834 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:03,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:03,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:03,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:03,838 DEBUG [Thread-160 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:19:03,839 DEBUG [Thread-160 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:03,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:03,842 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:03,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:19:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:19:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:19:03,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:03,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:03,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:19:03,861 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:03,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:03,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:03,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:03,868 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:19:03,875 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-02T14:19:03,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T14:19:03,881 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:03,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:03,884 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:19:03,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:03,885 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:03,892 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:19:03,898 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:03,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:03,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:03,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:03,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:19:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:03,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T14:19:03,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:03,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:03,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:03,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:03,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:03,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:03,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T14:19:04,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/521c3ccdbc4e4f0281911a19498c77dd is 50, key is test_row_0/A:col10/1733149143892/Put/seqid=0 2024-12-02T14:19:04,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T14:19:04,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:04,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:04,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:04,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741837_1013 (size=14341) 2024-12-02T14:19:04,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/521c3ccdbc4e4f0281911a19498c77dd 2024-12-02T14:19:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149204130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T14:19:04,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149204114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149204149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733149144059, max timestamp=1733149144142Num uniques 859; 2024-12-02T14:19:04,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:19:04,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149144059, max timestamp=1733149144142Num uniques 859; 2024-12-02T14:19:04,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:19:04,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149144059, max timestamp=1733149144142Num uniques 859; 2024-12-02T14:19:04,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:19:04,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T14:19:04,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/d42455bc8cdb48b7b307d4a87ed05706 is 50, key is test_row_0/B:col10/1733149143892/Put/seqid=0 2024-12-02T14:19:04,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:04,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:04,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:04,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741838_1014 (size=12001) 2024-12-02T14:19:04,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/d42455bc8cdb48b7b307d4a87ed05706 2024-12-02T14:19:04,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149204367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149204378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33296 deadline: 1733149204380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149204370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/951a59975b504d3c88e22c28c20e76c4 is 50, key is test_row_0/C:col10/1733149143892/Put/seqid=0 2024-12-02T14:19:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149204380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741839_1015 (size=12001) 2024-12-02T14:19:04,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/951a59975b504d3c88e22c28c20e76c4 2024-12-02T14:19:04,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/521c3ccdbc4e4f0281911a19498c77dd as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/521c3ccdbc4e4f0281911a19498c77dd 2024-12-02T14:19:04,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T14:19:04,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:04,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:04,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:04,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/521c3ccdbc4e4f0281911a19498c77dd, entries=200, sequenceid=12, filesize=14.0 K 2024-12-02T14:19:04,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/d42455bc8cdb48b7b307d4a87ed05706 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/d42455bc8cdb48b7b307d4a87ed05706 2024-12-02T14:19:04,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/d42455bc8cdb48b7b307d4a87ed05706, entries=150, sequenceid=12, filesize=11.7 K 2024-12-02T14:19:04,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/951a59975b504d3c88e22c28c20e76c4 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/951a59975b504d3c88e22c28c20e76c4 2024-12-02T14:19:04,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/951a59975b504d3c88e22c28c20e76c4, entries=150, sequenceid=12, filesize=11.7 K 2024-12-02T14:19:04,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T14:19:04,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 583ms, sequenceid=12, compaction requested=false 2024-12-02T14:19:04,527 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-02T14:19:04,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:04,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-02T14:19:04,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:04,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:04,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:04,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:04,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:04,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:04,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/c0c2f547548948ccbb64ec7fe84ef1dc is 50, key is test_row_0/A:col10/1733149144538/Put/seqid=0 2024-12-02T14:19:04,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33296 deadline: 1733149204569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149204574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T14:19:04,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:04,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:04,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:04,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741840_1016 (size=16681) 2024-12-02T14:19:04,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149204606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149204608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149204600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33296 deadline: 1733149204719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149204720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T14:19:04,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:04,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:04,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:04,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149204759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149204769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149204769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T14:19:04,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:04,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:04,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:04,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:04,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149204950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:04,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:04,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33296 deadline: 1733149204951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:04,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:04,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T14:19:05,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149205009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:05,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:05,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:05,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149205009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149205010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:05,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:05,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/c0c2f547548948ccbb64ec7fe84ef1dc 2024-12-02T14:19:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:05,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/c80ab0ebd09547dd93b37f0cab988d3d is 50, key is test_row_0/B:col10/1733149144538/Put/seqid=0 2024-12-02T14:19:05,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T14:19:05,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:05,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:05,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:05,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:05,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:05,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741841_1017 (size=12001) 2024-12-02T14:19:05,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T14:19:05,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:05,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:05,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:05,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:05,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:05,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33296 deadline: 1733149205280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:05,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:05,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:05,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:05,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149205280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:05,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:05,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149205330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:05,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149205340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:05,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:05,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:05,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:05,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149205340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:05,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T14:19:05,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:05,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:05,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:05,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:05,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:05,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/c80ab0ebd09547dd93b37f0cab988d3d 2024-12-02T14:19:05,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/11b36ffe8f464d8b8a9ef87d1c751baa is 50, key is test_row_0/C:col10/1733149144538/Put/seqid=0 2024-12-02T14:19:05,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T14:19:05,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:05,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:05,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:05,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:05,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:05,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741842_1018 (size=12001) 2024-12-02T14:19:05,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/11b36ffe8f464d8b8a9ef87d1c751baa 2024-12-02T14:19:05,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/c0c2f547548948ccbb64ec7fe84ef1dc as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c0c2f547548948ccbb64ec7fe84ef1dc 2024-12-02T14:19:05,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c0c2f547548948ccbb64ec7fe84ef1dc, entries=250, sequenceid=49, filesize=16.3 K 2024-12-02T14:19:05,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/c80ab0ebd09547dd93b37f0cab988d3d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/c80ab0ebd09547dd93b37f0cab988d3d 2024-12-02T14:19:05,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/c80ab0ebd09547dd93b37f0cab988d3d, entries=150, sequenceid=49, filesize=11.7 K 2024-12-02T14:19:05,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/11b36ffe8f464d8b8a9ef87d1c751baa as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/11b36ffe8f464d8b8a9ef87d1c751baa 2024-12-02T14:19:05,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/11b36ffe8f464d8b8a9ef87d1c751baa, entries=150, sequenceid=49, filesize=11.7 K 2024-12-02T14:19:05,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 1137ms, sequenceid=49, compaction requested=false 2024-12-02T14:19:05,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:05,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T14:19:05,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:05,716 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-02T14:19:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:05,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:05,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:05,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:05,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:05,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/3800a9e5b40945c694ca7a8506f240d9 is 50, key is test_row_1/A:col10/1733149144598/Put/seqid=0 2024-12-02T14:19:05,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741843_1019 (size=9657) 2024-12-02T14:19:05,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:05,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149145865, max timestamp=1733149145898Num uniques 859; 2024-12-02T14:19:05,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:19:05,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149145865, max timestamp=1733149145900Num uniques 859; 2024-12-02T14:19:05,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:19:05,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149145865, max timestamp=1733149145898Num uniques 859; 2024-12-02T14:19:05,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:19:05,966 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T14:19:05,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733149145950, max timestamp=1733149145976Num uniques 1018; 2024-12-02T14:19:05,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-02T14:19:05,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733149145964, max timestamp=1733149145976Num uniques 1018; 2024-12-02T14:19:05,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-12-02T14:19:06,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733149145964, max timestamp=1733149145976Num uniques 1018; 2024-12-02T14:19:06,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-02T14:19:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T14:19:06,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33296 deadline: 1733149206034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733149146011, max timestamp=1733149146046Num uniques 1027; 2024-12-02T14:19:06,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-02T14:19:06,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733149146011, max timestamp=1733149146046Num uniques 1009; 2024-12-02T14:19:06,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-12-02T14:19:06,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733149146011, max timestamp=1733149146046Num uniques 1127; 2024-12-02T14:19:06,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-12-02T14:19:06,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149206130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149206134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149206147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149206148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33296 deadline: 1733149206158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,197 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/3800a9e5b40945c694ca7a8506f240d9 2024-12-02T14:19:06,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/525b8a261c1641ae8193282d60781d07 is 50, key is test_row_1/B:col10/1733149144598/Put/seqid=0 2024-12-02T14:19:06,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149206259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149206259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149206269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149206279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741844_1020 (size=9657) 2024-12-02T14:19:06,296 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/525b8a261c1641ae8193282d60781d07 2024-12-02T14:19:06,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/ed5a32a7bdeb473ca5bc6f32202ed3df is 50, key is test_row_1/C:col10/1733149144598/Put/seqid=0 2024-12-02T14:19:06,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33296 deadline: 1733149206379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741845_1021 (size=9657) 2024-12-02T14:19:06,395 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/ed5a32a7bdeb473ca5bc6f32202ed3df 2024-12-02T14:19:06,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/3800a9e5b40945c694ca7a8506f240d9 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/3800a9e5b40945c694ca7a8506f240d9 2024-12-02T14:19:06,420 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/3800a9e5b40945c694ca7a8506f240d9, entries=100, sequenceid=58, filesize=9.4 K 2024-12-02T14:19:06,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/525b8a261c1641ae8193282d60781d07 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/525b8a261c1641ae8193282d60781d07 2024-12-02T14:19:06,441 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/525b8a261c1641ae8193282d60781d07, entries=100, sequenceid=58, filesize=9.4 K 2024-12-02T14:19:06,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/ed5a32a7bdeb473ca5bc6f32202ed3df as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/ed5a32a7bdeb473ca5bc6f32202ed3df 2024-12-02T14:19:06,457 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/ed5a32a7bdeb473ca5bc6f32202ed3df, entries=100, sequenceid=58, filesize=9.4 K 2024-12-02T14:19:06,459 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 743ms, sequenceid=58, compaction requested=true 2024-12-02T14:19:06,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:06,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:06,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-02T14:19:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-02T14:19:06,467 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-02T14:19:06,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5790 sec 2024-12-02T14:19:06,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.5990 sec 2024-12-02T14:19:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:06,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=147.60 KB heapSize=419.16 KB 2024-12-02T14:19:06,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:06,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T14:19:06,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:06,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T14:19:06,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:06,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T14:19:06,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/6db3ce214d214a3fa5be306ab019e64c is 50, key is test_row_0/A:col10/1733149146479/Put/seqid=0 2024-12-02T14:19:06,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149206499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149206510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149206510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149206512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741846_1022 (size=14391) 2024-12-02T14:19:06,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149206619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149206629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149206629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149206629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,677 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T14:19:06,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33296 deadline: 1733149206689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149206829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149206839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149206839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:06,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149206848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:06,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:06,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/6db3ce214d214a3fa5be306ab019e64c 2024-12-02T14:19:06,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/3750bb6fcb5544039eedf1835570fce7 is 50, key is test_row_0/B:col10/1733149146479/Put/seqid=0 2024-12-02T14:19:07,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741847_1023 (size=12051) 2024-12-02T14:19:07,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149207139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:07,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149207158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:07,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149207169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149207169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:07,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:07,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33296 deadline: 1733149207208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:07,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/3750bb6fcb5544039eedf1835570fce7 2024-12-02T14:19:07,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/da2d5efa98394bc3969f49fc89acfd6f is 50, key is test_row_0/C:col10/1733149146479/Put/seqid=0 2024-12-02T14:19:07,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741848_1024 (size=12051) 2024-12-02T14:19:07,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/da2d5efa98394bc3969f49fc89acfd6f 2024-12-02T14:19:07,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/6db3ce214d214a3fa5be306ab019e64c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/6db3ce214d214a3fa5be306ab019e64c 2024-12-02T14:19:07,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/6db3ce214d214a3fa5be306ab019e64c, entries=200, sequenceid=130, filesize=14.1 K 2024-12-02T14:19:07,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/3750bb6fcb5544039eedf1835570fce7 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/3750bb6fcb5544039eedf1835570fce7 2024-12-02T14:19:07,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/3750bb6fcb5544039eedf1835570fce7, entries=150, sequenceid=130, filesize=11.8 K 2024-12-02T14:19:07,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/da2d5efa98394bc3969f49fc89acfd6f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/da2d5efa98394bc3969f49fc89acfd6f 2024-12-02T14:19:07,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/da2d5efa98394bc3969f49fc89acfd6f, entries=150, sequenceid=130, filesize=11.8 K 2024-12-02T14:19:07,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.11 KB/429168, currentSize=33.54 KB/34350 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 1111ms, sequenceid=130, compaction requested=true 2024-12-02T14:19:07,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:07,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:07,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:07,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:07,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:07,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:07,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:07,610 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:07,614 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45710 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:07,616 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/B is initiating minor compaction (all files) 2024-12-02T14:19:07,616 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/B in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:07,617 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/d42455bc8cdb48b7b307d4a87ed05706, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/c80ab0ebd09547dd93b37f0cab988d3d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/525b8a261c1641ae8193282d60781d07, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/3750bb6fcb5544039eedf1835570fce7] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=44.6 K 2024-12-02T14:19:07,620 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting d42455bc8cdb48b7b307d4a87ed05706, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733149143892 2024-12-02T14:19:07,621 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:07,621 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting c80ab0ebd09547dd93b37f0cab988d3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733149144357 2024-12-02T14:19:07,622 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 525b8a261c1641ae8193282d60781d07, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733149144598 2024-12-02T14:19:07,623 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 3750bb6fcb5544039eedf1835570fce7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733149146139 2024-12-02T14:19:07,624 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55070 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:07,624 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/A is initiating minor compaction (all files) 2024-12-02T14:19:07,624 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/A in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:07,624 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/521c3ccdbc4e4f0281911a19498c77dd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c0c2f547548948ccbb64ec7fe84ef1dc, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/3800a9e5b40945c694ca7a8506f240d9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/6db3ce214d214a3fa5be306ab019e64c] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=53.8 K 2024-12-02T14:19:07,625 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 521c3ccdbc4e4f0281911a19498c77dd, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733149143848 2024-12-02T14:19:07,626 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0c2f547548948ccbb64ec7fe84ef1dc, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733149144357 2024-12-02T14:19:07,627 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3800a9e5b40945c694ca7a8506f240d9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733149144598 2024-12-02T14:19:07,628 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6db3ce214d214a3fa5be306ab019e64c, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733149146115 2024-12-02T14:19:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:07,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T14:19:07,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:07,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:07,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:07,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:07,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:07,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:07,680 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#A#compaction#12 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:07,681 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/ab88c1ff9b8548ec950ed50763449745 is 50, key is test_row_0/A:col10/1733149146479/Put/seqid=0 2024-12-02T14:19:07,685 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#B#compaction#13 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:07,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/64693bce5986433a982d13b6af03ff6b is 50, key is test_row_0/A:col10/1733149147671/Put/seqid=0 2024-12-02T14:19:07,691 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/a0c1865c20d14894a727716a18596522 is 50, key is test_row_0/B:col10/1733149146479/Put/seqid=0 2024-12-02T14:19:07,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741849_1025 (size=12189) 2024-12-02T14:19:07,725 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/ab88c1ff9b8548ec950ed50763449745 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ab88c1ff9b8548ec950ed50763449745 2024-12-02T14:19:07,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741850_1026 (size=12189) 2024-12-02T14:19:07,749 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/A of 4eaa960f4ddde5e2082f5ac5eba8cec8 into ab88c1ff9b8548ec950ed50763449745(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:07,749 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:07,755 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/A, priority=12, startTime=1733149147596; duration=0sec 2024-12-02T14:19:07,756 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:07,757 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:A 2024-12-02T14:19:07,757 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:07,761 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45710 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:07,761 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/C is initiating minor compaction (all files) 2024-12-02T14:19:07,762 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/C in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:07,762 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/951a59975b504d3c88e22c28c20e76c4, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/11b36ffe8f464d8b8a9ef87d1c751baa, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/ed5a32a7bdeb473ca5bc6f32202ed3df, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/da2d5efa98394bc3969f49fc89acfd6f] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=44.6 K 2024-12-02T14:19:07,764 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 951a59975b504d3c88e22c28c20e76c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733149143892 2024-12-02T14:19:07,766 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/a0c1865c20d14894a727716a18596522 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/a0c1865c20d14894a727716a18596522 2024-12-02T14:19:07,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741851_1027 (size=12147) 2024-12-02T14:19:07,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/64693bce5986433a982d13b6af03ff6b 2024-12-02T14:19:07,768 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11b36ffe8f464d8b8a9ef87d1c751baa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733149144357 2024-12-02T14:19:07,771 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed5a32a7bdeb473ca5bc6f32202ed3df, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733149144598 2024-12-02T14:19:07,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149207760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:07,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733149147736, max timestamp=1733149147764Num uniques 859; 2024-12-02T14:19:07,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T14:19:07,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733149147736, max timestamp=1733149147764Num uniques 859; 2024-12-02T14:19:07,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T14:19:07,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733149147736, max timestamp=1733149147764Num uniques 859; 2024-12-02T14:19:07,774 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting da2d5efa98394bc3969f49fc89acfd6f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733149146139 2024-12-02T14:19:07,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T14:19:07,782 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/B of 4eaa960f4ddde5e2082f5ac5eba8cec8 into a0c1865c20d14894a727716a18596522(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:07,782 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:07,782 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/B, priority=12, startTime=1733149147608; duration=0sec 2024-12-02T14:19:07,782 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:07,782 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:B 2024-12-02T14:19:07,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/b905e6ef6fbb4079a6fce303ec7f30e5 is 50, key is test_row_0/B:col10/1733149147671/Put/seqid=0 2024-12-02T14:19:07,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741852_1028 (size=9757) 2024-12-02T14:19:07,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/b905e6ef6fbb4079a6fce303ec7f30e5 2024-12-02T14:19:07,829 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#C#compaction#16 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:07,830 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/253b2a899518423bb14f66f9ccc18c6b is 50, key is test_row_0/C:col10/1733149146479/Put/seqid=0 2024-12-02T14:19:07,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/75182915b71245179f00ed51f8616878 is 50, key is test_row_0/C:col10/1733149147671/Put/seqid=0 2024-12-02T14:19:07,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149207837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:07,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149207848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:07,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149207850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:07,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:07,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149207878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:07,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:07,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741854_1030 (size=9757) 2024-12-02T14:19:07,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741853_1029 (size=12189) 2024-12-02T14:19:07,933 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/253b2a899518423bb14f66f9ccc18c6b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/253b2a899518423bb14f66f9ccc18c6b 2024-12-02T14:19:07,951 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/C of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 253b2a899518423bb14f66f9ccc18c6b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:07,952 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:07,952 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/C, priority=12, startTime=1733149147608; duration=0sec 2024-12-02T14:19:07,952 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:07,953 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:C 2024-12-02T14:19:07,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:07,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149207959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:07,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:07,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149207959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:07,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149207960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:08,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T14:19:08,059 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:08,059 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:08,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-02T14:19:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T14:19:08,064 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:08,070 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:08,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:08,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149208089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:08,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T14:19:08,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:08,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149208168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:08,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:08,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149208169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:08,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:08,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:08,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149208169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:08,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:08,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:08,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33296 deadline: 1733149208219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:08,224 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_1' in 4eaa960f4ddde5e2082f5ac5eba8cec8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:08,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T14:19:08,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:08,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:08,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:08,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:08,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:08,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:08,274 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T14:19:08,274 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T14:19:08,276 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T14:19:08,276 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T14:19:08,277 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-02T14:19:08,277 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-02T14:19:08,278 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-02T14:19:08,278 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-02T14:19:08,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/75182915b71245179f00ed51f8616878 2024-12-02T14:19:08,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/64693bce5986433a982d13b6af03ff6b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/64693bce5986433a982d13b6af03ff6b 2024-12-02T14:19:08,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/64693bce5986433a982d13b6af03ff6b, entries=150, sequenceid=140, filesize=11.9 K 2024-12-02T14:19:08,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/b905e6ef6fbb4079a6fce303ec7f30e5 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b905e6ef6fbb4079a6fce303ec7f30e5 2024-12-02T14:19:08,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b905e6ef6fbb4079a6fce303ec7f30e5, entries=100, sequenceid=140, filesize=9.5 K 2024-12-02T14:19:08,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/75182915b71245179f00ed51f8616878 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/75182915b71245179f00ed51f8616878 2024-12-02T14:19:08,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/75182915b71245179f00ed51f8616878, entries=100, sequenceid=140, filesize=9.5 K 2024-12-02T14:19:08,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=132.99 KB/136185 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 686ms, sequenceid=140, compaction requested=false 2024-12-02T14:19:08,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T14:19:08,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T14:19:08,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:08,385 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=132.99 KB heapSize=382.24 KB 2024-12-02T14:19:08,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:08,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:08,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:08,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:08,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:08,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:08,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:08,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/54c01548ad7541fe8793d662cafeb648 is 50, key is test_row_0/A:col10/1733149147832/Put/seqid=0 2024-12-02T14:19:08,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741855_1031 (size=12151) 2024-12-02T14:19:08,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149208466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:08,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149208479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:08,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:08,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149208480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149208480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:08,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149208579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:08,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T14:19:08,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:08,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149208789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:08,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:08,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:08,864 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/54c01548ad7541fe8793d662cafeb648 2024-12-02T14:19:08,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/ea23a104f0f54fb2aa3d9deee20dbe39 is 50, key is test_row_0/B:col10/1733149147832/Put/seqid=0 2024-12-02T14:19:08,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741856_1032 (size=12151) 2024-12-02T14:19:08,961 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/ea23a104f0f54fb2aa3d9deee20dbe39 2024-12-02T14:19:08,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/75262c3f425040f4b26f46c11aea7cc0 is 50, key is test_row_0/C:col10/1733149147832/Put/seqid=0 2024-12-02T14:19:09,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741857_1033 (size=12151) 2024-12-02T14:19:09,002 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/75262c3f425040f4b26f46c11aea7cc0 2024-12-02T14:19:09,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149208998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:09,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:09,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149208999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:09,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149208999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:09,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/54c01548ad7541fe8793d662cafeb648 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/54c01548ad7541fe8793d662cafeb648 2024-12-02T14:19:09,029 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/54c01548ad7541fe8793d662cafeb648, entries=150, sequenceid=180, filesize=11.9 K 2024-12-02T14:19:09,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/ea23a104f0f54fb2aa3d9deee20dbe39 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/ea23a104f0f54fb2aa3d9deee20dbe39 2024-12-02T14:19:09,046 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/ea23a104f0f54fb2aa3d9deee20dbe39, entries=150, sequenceid=180, filesize=11.9 K 2024-12-02T14:19:09,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/75262c3f425040f4b26f46c11aea7cc0 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/75262c3f425040f4b26f46c11aea7cc0 2024-12-02T14:19:09,064 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/75262c3f425040f4b26f46c11aea7cc0, entries=150, sequenceid=180, filesize=11.9 K 2024-12-02T14:19:09,066 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~132.99 KB/136185, heapSize ~382.20 KB/391368, currentSize=46.96 KB/48090 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 681ms, sequenceid=180, compaction requested=true 2024-12-02T14:19:09,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:09,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:09,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-02T14:19:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-02T14:19:09,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-02T14:19:09,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 996 msec 2024-12-02T14:19:09,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.0210 sec 2024-12-02T14:19:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:09,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T14:19:09,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:09,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:09,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:09,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:09,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:09,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:09,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/2b438b43cb81435584ef8b1990d1f31b is 50, key is test_row_0/A:col10/1733149148451/Put/seqid=0 2024-12-02T14:19:09,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741858_1034 (size=14541) 2024-12-02T14:19:09,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/2b438b43cb81435584ef8b1990d1f31b 2024-12-02T14:19:09,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/d7f70141cf1348b48baa5a9ddf4e969d is 50, key is test_row_0/B:col10/1733149148451/Put/seqid=0 2024-12-02T14:19:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T14:19:09,199 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:09,199 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-02T14:19:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T14:19:09,206 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:09,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741859_1035 (size=12151) 2024-12-02T14:19:09,207 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:09,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:09,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149149175, max timestamp=1733149149256Num uniques 859; 2024-12-02T14:19:09,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:09,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149149175, max timestamp=1733149149256Num uniques 859; 2024-12-02T14:19:09,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:19:09,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149149175, max timestamp=1733149149256Num uniques 859; 2024-12-02T14:19:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149209259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:09,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:19:09,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:19:09,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:09,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:09,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T14:19:09,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T14:19:09,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:09,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:09,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:09,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:09,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:09,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149209498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:09,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:09,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:09,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T14:19:09,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T14:19:09,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:09,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:09,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:09,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:09,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:09,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/d7f70141cf1348b48baa5a9ddf4e969d 2024-12-02T14:19:09,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149209618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:09,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/05ff48dc5b134771b0bb7c19f1ba7bba is 50, key is test_row_0/C:col10/1733149148451/Put/seqid=0 2024-12-02T14:19:09,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T14:19:09,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741860_1036 (size=12151) 2024-12-02T14:19:09,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:09,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:09,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:09,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:09,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:09,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/05ff48dc5b134771b0bb7c19f1ba7bba 2024-12-02T14:19:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:09,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/2b438b43cb81435584ef8b1990d1f31b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/2b438b43cb81435584ef8b1990d1f31b 2024-12-02T14:19:09,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/2b438b43cb81435584ef8b1990d1f31b, entries=200, sequenceid=191, filesize=14.2 K 2024-12-02T14:19:09,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/d7f70141cf1348b48baa5a9ddf4e969d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/d7f70141cf1348b48baa5a9ddf4e969d 2024-12-02T14:19:09,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/d7f70141cf1348b48baa5a9ddf4e969d, entries=150, sequenceid=191, filesize=11.9 K 2024-12-02T14:19:09,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/05ff48dc5b134771b0bb7c19f1ba7bba as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/05ff48dc5b134771b0bb7c19f1ba7bba 2024-12-02T14:19:09,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/05ff48dc5b134771b0bb7c19f1ba7bba, entries=150, sequenceid=191, filesize=11.9 K 2024-12-02T14:19:09,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 635ms, sequenceid=191, compaction requested=true 2024-12-02T14:19:09,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:09,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:09,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:09,740 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:09,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:09,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:09,740 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:09,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:09,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:09,743 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51028 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:09,743 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/A is initiating minor compaction (all files) 2024-12-02T14:19:09,743 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/A in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:09,743 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ab88c1ff9b8548ec950ed50763449745, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/64693bce5986433a982d13b6af03ff6b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/54c01548ad7541fe8793d662cafeb648, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/2b438b43cb81435584ef8b1990d1f31b] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=49.8 K 2024-12-02T14:19:09,744 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab88c1ff9b8548ec950ed50763449745, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733149146139 2024-12-02T14:19:09,745 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46248 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:09,745 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/B is initiating minor compaction (all files) 2024-12-02T14:19:09,745 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/B in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:09,745 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/a0c1865c20d14894a727716a18596522, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b905e6ef6fbb4079a6fce303ec7f30e5, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/ea23a104f0f54fb2aa3d9deee20dbe39, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/d7f70141cf1348b48baa5a9ddf4e969d] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=45.2 K 2024-12-02T14:19:09,745 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64693bce5986433a982d13b6af03ff6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733149146495 2024-12-02T14:19:09,746 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting a0c1865c20d14894a727716a18596522, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733149146139 2024-12-02T14:19:09,746 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54c01548ad7541fe8793d662cafeb648, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733149147830 2024-12-02T14:19:09,746 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting b905e6ef6fbb4079a6fce303ec7f30e5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733149147661 2024-12-02T14:19:09,747 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting ea23a104f0f54fb2aa3d9deee20dbe39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733149147830 2024-12-02T14:19:09,747 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting d7f70141cf1348b48baa5a9ddf4e969d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733149148451 2024-12-02T14:19:09,748 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b438b43cb81435584ef8b1990d1f31b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733149148422 2024-12-02T14:19:09,773 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#B#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:09,774 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/f5f2801c4fc94e95ab3350249489dd29 is 50, key is test_row_0/B:col10/1733149148451/Put/seqid=0 2024-12-02T14:19:09,781 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#A#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:09,781 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/b7479d9dec994fcb8e17bdfcccb09a13 is 50, key is test_row_0/A:col10/1733149148451/Put/seqid=0 2024-12-02T14:19:09,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741861_1037 (size=12425) 2024-12-02T14:19:09,808 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/f5f2801c4fc94e95ab3350249489dd29 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/f5f2801c4fc94e95ab3350249489dd29 2024-12-02T14:19:09,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741862_1038 (size=12425) 2024-12-02T14:19:09,821 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/B of 4eaa960f4ddde5e2082f5ac5eba8cec8 into f5f2801c4fc94e95ab3350249489dd29(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:09,822 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:09,822 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/B, priority=12, startTime=1733149149740; duration=0sec 2024-12-02T14:19:09,822 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:09,822 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:B 2024-12-02T14:19:09,822 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:09,828 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/b7479d9dec994fcb8e17bdfcccb09a13 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/b7479d9dec994fcb8e17bdfcccb09a13 2024-12-02T14:19:09,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T14:19:09,830 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46248 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:09,830 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/C is initiating minor compaction (all files) 2024-12-02T14:19:09,830 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/C in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:09,830 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/253b2a899518423bb14f66f9ccc18c6b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/75182915b71245179f00ed51f8616878, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/75262c3f425040f4b26f46c11aea7cc0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/05ff48dc5b134771b0bb7c19f1ba7bba] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=45.2 K 2024-12-02T14:19:09,831 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 253b2a899518423bb14f66f9ccc18c6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733149146139 2024-12-02T14:19:09,831 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 75182915b71245179f00ed51f8616878, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733149147661 2024-12-02T14:19:09,834 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 75262c3f425040f4b26f46c11aea7cc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733149147830 2024-12-02T14:19:09,835 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ff48dc5b134771b0bb7c19f1ba7bba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733149148451 2024-12-02T14:19:09,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T14:19:09,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:09,837 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-02T14:19:09,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:09,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:09,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:09,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:09,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:09,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:09,840 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/A of 4eaa960f4ddde5e2082f5ac5eba8cec8 into b7479d9dec994fcb8e17bdfcccb09a13(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:09,840 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:09,840 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/A, priority=12, startTime=1733149149740; duration=0sec 2024-12-02T14:19:09,840 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:09,840 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:A 2024-12-02T14:19:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:09,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:09,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/437af98b938743b488cb8913454f2d15 is 50, key is test_row_0/A:col10/1733149149475/Put/seqid=0 2024-12-02T14:19:09,867 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#C#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:09,868 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/3857f165dfd3425cb0327b590994dc1f is 50, key is test_row_0/C:col10/1733149148451/Put/seqid=0 2024-12-02T14:19:09,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741863_1039 (size=12151) 2024-12-02T14:19:09,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741864_1040 (size=12425) 2024-12-02T14:19:09,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149209887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:09,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:09,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:09,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:09,907 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/3857f165dfd3425cb0327b590994dc1f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/3857f165dfd3425cb0327b590994dc1f 2024-12-02T14:19:09,915 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/C of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 3857f165dfd3425cb0327b590994dc1f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:09,915 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:09,915 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/C, priority=12, startTime=1733149149740; duration=0sec 2024-12-02T14:19:09,916 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:09,916 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:C 2024-12-02T14:19:10,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:10,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149209999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:10,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:10,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:10,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149210019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149210018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:10,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149210019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:10,021 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 4eaa960f4ddde5e2082f5ac5eba8cec8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:10,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:10,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:10,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:10,022 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 4eaa960f4ddde5e2082f5ac5eba8cec8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:10,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:10,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:10,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:10,023 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 4eaa960f4ddde5e2082f5ac5eba8cec8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:10,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:10,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149210208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:10,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:10,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33296 deadline: 1733149210229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:10,234 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_1' in 4eaa960f4ddde5e2082f5ac5eba8cec8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:10,284 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/437af98b938743b488cb8913454f2d15 2024-12-02T14:19:10,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/da13b012c9fc425f8e7b635ee4760717 is 50, key is test_row_0/B:col10/1733149149475/Put/seqid=0 2024-12-02T14:19:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T14:19:10,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741865_1041 (size=12151) 2024-12-02T14:19:10,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149210519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:10,757 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/da13b012c9fc425f8e7b635ee4760717 2024-12-02T14:19:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/d6c8fbd16acc45889af586ec1cef26c0 is 50, key is test_row_0/C:col10/1733149149475/Put/seqid=0 2024-12-02T14:19:10,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741866_1042 (size=12151) 2024-12-02T14:19:10,818 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/d6c8fbd16acc45889af586ec1cef26c0 2024-12-02T14:19:10,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/437af98b938743b488cb8913454f2d15 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/437af98b938743b488cb8913454f2d15 2024-12-02T14:19:10,838 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/437af98b938743b488cb8913454f2d15, entries=150, sequenceid=228, filesize=11.9 K 2024-12-02T14:19:10,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/da13b012c9fc425f8e7b635ee4760717 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/da13b012c9fc425f8e7b635ee4760717 2024-12-02T14:19:10,858 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/da13b012c9fc425f8e7b635ee4760717, entries=150, sequenceid=228, filesize=11.9 K 2024-12-02T14:19:10,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/d6c8fbd16acc45889af586ec1cef26c0 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/d6c8fbd16acc45889af586ec1cef26c0 2024-12-02T14:19:10,872 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/d6c8fbd16acc45889af586ec1cef26c0, entries=150, sequenceid=228, filesize=11.9 K 2024-12-02T14:19:10,873 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 1036ms, sequenceid=228, compaction requested=false 2024-12-02T14:19:10,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:10,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:10,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-02T14:19:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-02T14:19:10,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-02T14:19:10,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6680 sec 2024-12-02T14:19:10,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.6870 sec 2024-12-02T14:19:11,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T14:19:11,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:11,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:11,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:11,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:11,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:11,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:11,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/ebf614b3659847c1a775cc1e91201053 is 50, key is test_row_0/A:col10/1733149149885/Put/seqid=0 2024-12-02T14:19:11,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741867_1043 (size=12151) 2024-12-02T14:19:11,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/ebf614b3659847c1a775cc1e91201053 2024-12-02T14:19:11,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/e6a9bc824b89471cb22d4d2b96787062 is 50, key is test_row_0/B:col10/1733149149885/Put/seqid=0 2024-12-02T14:19:11,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741868_1044 (size=12151) 2024-12-02T14:19:11,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/e6a9bc824b89471cb22d4d2b96787062 2024-12-02T14:19:11,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/1a9fd5eb1fc346b5a930b8448706efa2 is 50, key is test_row_0/C:col10/1733149149885/Put/seqid=0 2024-12-02T14:19:11,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149211217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:11,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:11,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:11,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:11,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741869_1045 (size=12151) 2024-12-02T14:19:11,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149211328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:11,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:11,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:11,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T14:19:11,349 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:11,350 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-02T14:19:11,363 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:11,367 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:11,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T14:19:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T14:19:11,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T14:19:11,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:11,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:11,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:11,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:11,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:11,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:11,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149211538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:11,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:11,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:11,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:11,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/1a9fd5eb1fc346b5a930b8448706efa2 2024-12-02T14:19:11,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/ebf614b3659847c1a775cc1e91201053 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ebf614b3659847c1a775cc1e91201053 2024-12-02T14:19:11,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ebf614b3659847c1a775cc1e91201053, entries=150, sequenceid=242, filesize=11.9 K 2024-12-02T14:19:11,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/e6a9bc824b89471cb22d4d2b96787062 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e6a9bc824b89471cb22d4d2b96787062 2024-12-02T14:19:11,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e6a9bc824b89471cb22d4d2b96787062, entries=150, sequenceid=242, filesize=11.9 K 2024-12-02T14:19:11,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/1a9fd5eb1fc346b5a930b8448706efa2 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/1a9fd5eb1fc346b5a930b8448706efa2 2024-12-02T14:19:11,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T14:19:11,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:11,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:11,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:11,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:11,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T14:19:11,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/1a9fd5eb1fc346b5a930b8448706efa2, entries=150, sequenceid=242, filesize=11.9 K 2024-12-02T14:19:11,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 662ms, sequenceid=242, compaction requested=true 2024-12-02T14:19:11,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:11,693 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:11,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:11,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:11,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:11,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:11,694 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:11,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:11,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:11,695 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:11,695 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/A is initiating minor compaction (all files) 2024-12-02T14:19:11,695 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/A in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:11,695 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/b7479d9dec994fcb8e17bdfcccb09a13, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/437af98b938743b488cb8913454f2d15, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ebf614b3659847c1a775cc1e91201053] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=35.9 K 2024-12-02T14:19:11,696 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:11,696 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7479d9dec994fcb8e17bdfcccb09a13, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733149148451 2024-12-02T14:19:11,696 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/B is initiating minor compaction (all files) 2024-12-02T14:19:11,696 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/B in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:11,696 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/f5f2801c4fc94e95ab3350249489dd29, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/da13b012c9fc425f8e7b635ee4760717, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e6a9bc824b89471cb22d4d2b96787062] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=35.9 K 2024-12-02T14:19:11,696 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 437af98b938743b488cb8913454f2d15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733149149467 2024-12-02T14:19:11,697 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting f5f2801c4fc94e95ab3350249489dd29, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733149148451 2024-12-02T14:19:11,697 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebf614b3659847c1a775cc1e91201053, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733149149863 2024-12-02T14:19:11,698 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting da13b012c9fc425f8e7b635ee4760717, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733149149467 2024-12-02T14:19:11,699 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting e6a9bc824b89471cb22d4d2b96787062, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733149149863 2024-12-02T14:19:11,712 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#A#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:11,712 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/7d31eef675d3403388480bc9662e8dfb is 50, key is test_row_0/A:col10/1733149149885/Put/seqid=0 2024-12-02T14:19:11,715 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#B#compaction#34 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:11,716 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/556a5387681a44c586b1fca4af963fcd is 50, key is test_row_0/B:col10/1733149149885/Put/seqid=0 2024-12-02T14:19:11,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741871_1047 (size=12527) 2024-12-02T14:19:11,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741870_1046 (size=12527) 2024-12-02T14:19:11,758 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/556a5387681a44c586b1fca4af963fcd as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/556a5387681a44c586b1fca4af963fcd 2024-12-02T14:19:11,772 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/B of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 556a5387681a44c586b1fca4af963fcd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:11,772 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:11,772 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/B, priority=13, startTime=1733149151694; duration=0sec 2024-12-02T14:19:11,773 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:11,773 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:B 2024-12-02T14:19:11,773 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:11,780 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:11,780 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/C is initiating minor compaction (all files) 2024-12-02T14:19:11,780 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/C in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:11,780 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/3857f165dfd3425cb0327b590994dc1f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/d6c8fbd16acc45889af586ec1cef26c0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/1a9fd5eb1fc346b5a930b8448706efa2] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=35.9 K 2024-12-02T14:19:11,784 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 3857f165dfd3425cb0327b590994dc1f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733149148451 2024-12-02T14:19:11,791 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting d6c8fbd16acc45889af586ec1cef26c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733149149467 2024-12-02T14:19:11,793 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a9fd5eb1fc346b5a930b8448706efa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733149149863 2024-12-02T14:19:11,808 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#C#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:11,809 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/2b92a56044bb4644beed7d3677603759 is 50, key is test_row_0/C:col10/1733149149885/Put/seqid=0 2024-12-02T14:19:11,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741872_1048 (size=12527) 2024-12-02T14:19:11,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T14:19:11,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:11,836 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T14:19:11,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:11,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:11,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:11,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:11,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:11,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:11,842 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/2b92a56044bb4644beed7d3677603759 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2b92a56044bb4644beed7d3677603759 2024-12-02T14:19:11,853 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/C of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 2b92a56044bb4644beed7d3677603759(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:11,853 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:11,853 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/C, priority=13, startTime=1733149151694; duration=0sec 2024-12-02T14:19:11,853 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:11,853 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:C 2024-12-02T14:19:11,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/c1185de9bfd34d29a88ca7063512e40b is 50, key is test_row_0/A:col10/1733149151188/Put/seqid=0 2024-12-02T14:19:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:11,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:11,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741873_1049 (size=12301) 2024-12-02T14:19:11,912 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/c1185de9bfd34d29a88ca7063512e40b 2024-12-02T14:19:11,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/c10b5e535c2a45c6a464dc55aaae97e7 is 50, key is test_row_0/B:col10/1733149151188/Put/seqid=0 2024-12-02T14:19:11,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741874_1050 (size=12301) 2024-12-02T14:19:11,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149211977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T14:19:12,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149212039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:12,051 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 4eaa960f4ddde5e2082f5ac5eba8cec8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:12,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:12,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149212039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:12,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:12,054 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 4eaa960f4ddde5e2082f5ac5eba8cec8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:12,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:12,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:12,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149212049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:12,059 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 4eaa960f4ddde5e2082f5ac5eba8cec8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:12,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149212089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:12,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:12,143 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/7d31eef675d3403388480bc9662e8dfb as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/7d31eef675d3403388480bc9662e8dfb 2024-12-02T14:19:12,155 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/A of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 7d31eef675d3403388480bc9662e8dfb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:12,155 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:12,155 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/A, priority=13, startTime=1733149151693; duration=0sec 2024-12-02T14:19:12,155 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:12,155 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:A 2024-12-02T14:19:12,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149212308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:12,364 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/c10b5e535c2a45c6a464dc55aaae97e7 2024-12-02T14:19:12,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/b390a727326c40aa9171896820d53480 is 50, key is test_row_0/C:col10/1733149151188/Put/seqid=0 2024-12-02T14:19:12,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741875_1051 (size=12301) 2024-12-02T14:19:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T14:19:12,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149212646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:12,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:12,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:12,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:12,851 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/b390a727326c40aa9171896820d53480 2024-12-02T14:19:12,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/c1185de9bfd34d29a88ca7063512e40b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c1185de9bfd34d29a88ca7063512e40b 2024-12-02T14:19:12,873 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c1185de9bfd34d29a88ca7063512e40b, entries=150, sequenceid=263, filesize=12.0 K 2024-12-02T14:19:12,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/c10b5e535c2a45c6a464dc55aaae97e7 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/c10b5e535c2a45c6a464dc55aaae97e7 2024-12-02T14:19:12,894 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/c10b5e535c2a45c6a464dc55aaae97e7, entries=150, sequenceid=263, filesize=12.0 K 2024-12-02T14:19:12,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/b390a727326c40aa9171896820d53480 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b390a727326c40aa9171896820d53480 2024-12-02T14:19:12,908 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b390a727326c40aa9171896820d53480, entries=150, sequenceid=263, filesize=12.0 K 2024-12-02T14:19:12,910 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 1073ms, sequenceid=263, compaction requested=false 2024-12-02T14:19:12,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:12,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:12,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-02T14:19:12,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-02T14:19:12,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-02T14:19:12,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5450 sec 2024-12-02T14:19:12,919 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.5670 sec 2024-12-02T14:19:13,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:13,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T14:19:13,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:13,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:13,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:13,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:13,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:13,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:13,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/b875a627322746fc94b4a007515a2ae8 is 50, key is test_row_0/A:col10/1733149153171/Put/seqid=0 2024-12-02T14:19:13,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741876_1052 (size=14741) 2024-12-02T14:19:13,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149213389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:13,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:13,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:13,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T14:19:13,520 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:13,522 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:13,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:13,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149213519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:13,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:13,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:13,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-02T14:19:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T14:19:13,528 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:13,530 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:13,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T14:19:13,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/b875a627322746fc94b4a007515a2ae8 2024-12-02T14:19:13,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/e1eb1bb7298b4ddd8b98bf72fe428fd6 is 50, key is test_row_0/B:col10/1733149153171/Put/seqid=0 2024-12-02T14:19:13,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T14:19:13,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:13,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:13,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:13,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:13,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:13,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741877_1053 (size=12301) 2024-12-02T14:19:13,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/e1eb1bb7298b4ddd8b98bf72fe428fd6 2024-12-02T14:19:13,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:13,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149213729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:13,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:13,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:13,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:13,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/2e7f57b562a643c889d3f4b15dd3e67a is 50, key is test_row_0/C:col10/1733149153171/Put/seqid=0 2024-12-02T14:19:13,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741878_1054 (size=12301) 2024-12-02T14:19:13,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/2e7f57b562a643c889d3f4b15dd3e67a 2024-12-02T14:19:13,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/b875a627322746fc94b4a007515a2ae8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/b875a627322746fc94b4a007515a2ae8 2024-12-02T14:19:13,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/b875a627322746fc94b4a007515a2ae8, entries=200, sequenceid=278, filesize=14.4 K 2024-12-02T14:19:13,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/e1eb1bb7298b4ddd8b98bf72fe428fd6 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e1eb1bb7298b4ddd8b98bf72fe428fd6 2024-12-02T14:19:13,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e1eb1bb7298b4ddd8b98bf72fe428fd6, entries=150, sequenceid=278, filesize=12.0 K 2024-12-02T14:19:13,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/2e7f57b562a643c889d3f4b15dd3e67a as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2e7f57b562a643c889d3f4b15dd3e67a 2024-12-02T14:19:13,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2e7f57b562a643c889d3f4b15dd3e67a, entries=150, sequenceid=278, filesize=12.0 K 2024-12-02T14:19:13,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 673ms, sequenceid=278, compaction requested=true 2024-12-02T14:19:13,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:13,848 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:13,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T14:19:13,850 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39569 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:13,850 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/A is initiating minor compaction (all files) 2024-12-02T14:19:13,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T14:19:13,851 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/A in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:13,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:13,851 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/7d31eef675d3403388480bc9662e8dfb, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c1185de9bfd34d29a88ca7063512e40b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/b875a627322746fc94b4a007515a2ae8] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=38.6 K 2024-12-02T14:19:13,851 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T14:19:13,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:13,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:13,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:13,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:13,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:13,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:13,852 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d31eef675d3403388480bc9662e8dfb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733149149863 2024-12-02T14:19:13,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:13,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:13,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:13,853 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:13,854 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1185de9bfd34d29a88ca7063512e40b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733149151188 2024-12-02T14:19:13,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:13,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:13,855 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting b875a627322746fc94b4a007515a2ae8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733149151927 2024-12-02T14:19:13,855 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:13,855 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/B is initiating minor compaction (all files) 2024-12-02T14:19:13,855 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/B in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:13,855 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/556a5387681a44c586b1fca4af963fcd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/c10b5e535c2a45c6a464dc55aaae97e7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e1eb1bb7298b4ddd8b98bf72fe428fd6] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=36.3 K 2024-12-02T14:19:13,856 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 556a5387681a44c586b1fca4af963fcd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733149149863 2024-12-02T14:19:13,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:13,857 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting c10b5e535c2a45c6a464dc55aaae97e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733149151188 2024-12-02T14:19:13,857 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting e1eb1bb7298b4ddd8b98bf72fe428fd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733149151958 2024-12-02T14:19:13,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/ffd108b0695540c895db489a2b443256 is 50, key is test_row_0/A:col10/1733149153341/Put/seqid=0 2024-12-02T14:19:13,892 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#B#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:13,892 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/925e8d4a2c944312a0f4685014523a3e is 50, key is test_row_0/B:col10/1733149153171/Put/seqid=0 2024-12-02T14:19:13,896 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#A#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:13,897 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/63f6afa2d0534ec9bf593f6114e13b22 is 50, key is test_row_0/A:col10/1733149153171/Put/seqid=0 2024-12-02T14:19:13,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741880_1056 (size=12779) 2024-12-02T14:19:13,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741879_1055 (size=12301) 2024-12-02T14:19:13,965 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/ffd108b0695540c895db489a2b443256 2024-12-02T14:19:13,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741881_1057 (size=12779) 2024-12-02T14:19:14,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/e89b002617b94771b049945db70dd978 is 50, key is test_row_0/B:col10/1733149153341/Put/seqid=0 2024-12-02T14:19:14,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741882_1058 (size=12301) 2024-12-02T14:19:14,033 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/e89b002617b94771b049945db70dd978 2024-12-02T14:19:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:14,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:14,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/bb3b88843a9146aa8afb39fe89b66384 is 50, key is test_row_0/C:col10/1733149153341/Put/seqid=0 2024-12-02T14:19:14,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741883_1059 (size=12301) 2024-12-02T14:19:14,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149214092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T14:19:14,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149214208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:14,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:14,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:14,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:14,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33296 deadline: 1733149214249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:14,253 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_1' in 4eaa960f4ddde5e2082f5ac5eba8cec8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:14,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:14,376 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/925e8d4a2c944312a0f4685014523a3e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/925e8d4a2c944312a0f4685014523a3e 2024-12-02T14:19:14,391 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/B of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 925e8d4a2c944312a0f4685014523a3e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:14,391 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:14,391 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/B, priority=13, startTime=1733149153853; duration=0sec 2024-12-02T14:19:14,392 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:14,392 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:B 2024-12-02T14:19:14,392 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:14,401 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:14,401 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/C is initiating minor compaction (all files) 2024-12-02T14:19:14,401 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/C in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:14,402 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2b92a56044bb4644beed7d3677603759, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b390a727326c40aa9171896820d53480, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2e7f57b562a643c889d3f4b15dd3e67a] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=36.3 K 2024-12-02T14:19:14,405 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b92a56044bb4644beed7d3677603759, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733149149863 2024-12-02T14:19:14,408 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting b390a727326c40aa9171896820d53480, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733149151188 2024-12-02T14:19:14,409 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e7f57b562a643c889d3f4b15dd3e67a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733149151958 2024-12-02T14:19:14,425 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/63f6afa2d0534ec9bf593f6114e13b22 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/63f6afa2d0534ec9bf593f6114e13b22 2024-12-02T14:19:14,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149214419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:14,431 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#C#compaction#47 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:14,431 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/2cd0dd2b7a2b4d83bb816cc64ec1f115 is 50, key is test_row_0/C:col10/1733149153171/Put/seqid=0 2024-12-02T14:19:14,438 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/A of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 63f6afa2d0534ec9bf593f6114e13b22(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:14,438 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:14,438 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/A, priority=13, startTime=1733149153848; duration=0sec 2024-12-02T14:19:14,438 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:14,438 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:A 2024-12-02T14:19:14,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741884_1060 (size=12779) 2024-12-02T14:19:14,477 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/bb3b88843a9146aa8afb39fe89b66384 2024-12-02T14:19:14,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/ffd108b0695540c895db489a2b443256 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ffd108b0695540c895db489a2b443256 2024-12-02T14:19:14,500 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ffd108b0695540c895db489a2b443256, entries=150, sequenceid=297, filesize=12.0 K 2024-12-02T14:19:14,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/e89b002617b94771b049945db70dd978 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e89b002617b94771b049945db70dd978 2024-12-02T14:19:14,502 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/2cd0dd2b7a2b4d83bb816cc64ec1f115 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2cd0dd2b7a2b4d83bb816cc64ec1f115 2024-12-02T14:19:14,515 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e89b002617b94771b049945db70dd978, entries=150, sequenceid=297, filesize=12.0 K 2024-12-02T14:19:14,519 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/C of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 2cd0dd2b7a2b4d83bb816cc64ec1f115(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:14,519 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:14,519 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/C, priority=13, startTime=1733149153855; duration=0sec 2024-12-02T14:19:14,519 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:14,519 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:C 2024-12-02T14:19:14,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/bb3b88843a9146aa8afb39fe89b66384 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/bb3b88843a9146aa8afb39fe89b66384 2024-12-02T14:19:14,541 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/bb3b88843a9146aa8afb39fe89b66384, entries=150, sequenceid=297, filesize=12.0 K 2024-12-02T14:19:14,542 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 691ms, sequenceid=297, compaction requested=false 2024-12-02T14:19:14,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:14,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:14,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-02T14:19:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-02T14:19:14,550 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-02T14:19:14,550 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0150 sec 2024-12-02T14:19:14,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.0320 sec 2024-12-02T14:19:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T14:19:14,669 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:14,672 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-02T14:19:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T14:19:14,676 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:14,677 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:14,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:14,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T14:19:14,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:14,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:14,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:14,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:14,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:14,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:14,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/615f00dba520419c96a253db025ca83c is 50, key is test_row_0/A:col10/1733149154084/Put/seqid=0 2024-12-02T14:19:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T14:19:14,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741885_1061 (size=14741) 2024-12-02T14:19:14,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/615f00dba520419c96a253db025ca83c 2024-12-02T14:19:14,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T14:19:14,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/43f9950d2640433390b7e2731dd43892 is 50, key is test_row_0/B:col10/1733149154084/Put/seqid=0 2024-12-02T14:19:14,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:14,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:14,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:14,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:14,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:14,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741886_1062 (size=12301) 2024-12-02T14:19:14,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149214871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:14,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:14,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149214979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T14:19:14,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T14:19:14,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:14,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:14,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:14,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:14,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:15,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T14:19:15,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:15,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:15,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:15,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:15,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:15,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149215189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:15,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/43f9950d2640433390b7e2731dd43892 2024-12-02T14:19:15,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/21b6fceafea74ede99138eab4882a180 is 50, key is test_row_0/C:col10/1733149154084/Put/seqid=0 2024-12-02T14:19:15,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741887_1063 (size=12301) 2024-12-02T14:19:15,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/21b6fceafea74ede99138eab4882a180 2024-12-02T14:19:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T14:19:15,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T14:19:15,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:15,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:15,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:15,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:15,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:15,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/615f00dba520419c96a253db025ca83c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/615f00dba520419c96a253db025ca83c 2024-12-02T14:19:15,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/615f00dba520419c96a253db025ca83c, entries=200, sequenceid=314, filesize=14.4 K 2024-12-02T14:19:15,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/43f9950d2640433390b7e2731dd43892 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/43f9950d2640433390b7e2731dd43892 2024-12-02T14:19:15,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/43f9950d2640433390b7e2731dd43892, entries=150, sequenceid=314, filesize=12.0 K 2024-12-02T14:19:15,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/21b6fceafea74ede99138eab4882a180 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/21b6fceafea74ede99138eab4882a180 2024-12-02T14:19:15,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/21b6fceafea74ede99138eab4882a180, entries=150, sequenceid=314, filesize=12.0 K 2024-12-02T14:19:15,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 614ms, sequenceid=314, compaction requested=true 2024-12-02T14:19:15,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:15,356 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:15,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:15,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:15,357 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:15,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:15,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:15,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:15,358 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:15,358 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/A is initiating minor compaction (all files) 2024-12-02T14:19:15,358 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/A in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:15,359 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/63f6afa2d0534ec9bf593f6114e13b22, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ffd108b0695540c895db489a2b443256, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/615f00dba520419c96a253db025ca83c] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=38.9 K 2024-12-02T14:19:15,361 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:15,361 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/B is initiating minor compaction (all files) 2024-12-02T14:19:15,361 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/B in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:15,361 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/925e8d4a2c944312a0f4685014523a3e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e89b002617b94771b049945db70dd978, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/43f9950d2640433390b7e2731dd43892] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=36.5 K 2024-12-02T14:19:15,362 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63f6afa2d0534ec9bf593f6114e13b22, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733149151958 2024-12-02T14:19:15,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:15,363 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffd108b0695540c895db489a2b443256, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733149153341 2024-12-02T14:19:15,363 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 925e8d4a2c944312a0f4685014523a3e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733149151958 2024-12-02T14:19:15,363 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting e89b002617b94771b049945db70dd978, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733149153341 2024-12-02T14:19:15,363 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 615f00dba520419c96a253db025ca83c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733149154073 2024-12-02T14:19:15,364 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 43f9950d2640433390b7e2731dd43892, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733149154073 2024-12-02T14:19:15,383 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:15,384 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/bdcba523237944b0aaee6ed1d0bbec85 is 50, key is test_row_0/B:col10/1733149154084/Put/seqid=0 2024-12-02T14:19:15,386 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#A#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:15,387 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/84b7332e744d4229a9b28fbf34ca4195 is 50, key is test_row_0/A:col10/1733149154084/Put/seqid=0 2024-12-02T14:19:15,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741889_1065 (size=12881) 2024-12-02T14:19:15,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741888_1064 (size=12881) 2024-12-02T14:19:15,426 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/bdcba523237944b0aaee6ed1d0bbec85 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/bdcba523237944b0aaee6ed1d0bbec85 2024-12-02T14:19:15,446 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/B of 4eaa960f4ddde5e2082f5ac5eba8cec8 into bdcba523237944b0aaee6ed1d0bbec85(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:15,446 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:15,446 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/B, priority=13, startTime=1733149155357; duration=0sec 2024-12-02T14:19:15,446 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:15,446 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:B 2024-12-02T14:19:15,446 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:15,451 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:15,452 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/C is initiating minor compaction (all files) 2024-12-02T14:19:15,452 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/C in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:15,452 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2cd0dd2b7a2b4d83bb816cc64ec1f115, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/bb3b88843a9146aa8afb39fe89b66384, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/21b6fceafea74ede99138eab4882a180] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=36.5 K 2024-12-02T14:19:15,452 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cd0dd2b7a2b4d83bb816cc64ec1f115, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733149151958 2024-12-02T14:19:15,453 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting bb3b88843a9146aa8afb39fe89b66384, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733149153341 2024-12-02T14:19:15,453 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 21b6fceafea74ede99138eab4882a180, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733149154073 2024-12-02T14:19:15,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T14:19:15,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:15,468 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T14:19:15,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:15,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:15,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:15,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:15,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:15,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:15,478 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#C#compaction#53 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:15,479 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/a9de6b9ccfb1433cb5e01a5d8e9924eb is 50, key is test_row_0/C:col10/1733149154084/Put/seqid=0 2024-12-02T14:19:15,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/daf2c8536a484bdcb5da63128a99e71d is 50, key is test_row_0/A:col10/1733149154836/Put/seqid=0 2024-12-02T14:19:15,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:15,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741890_1066 (size=12881) 2024-12-02T14:19:15,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741891_1067 (size=12301) 2024-12-02T14:19:15,534 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/daf2c8536a484bdcb5da63128a99e71d 2024-12-02T14:19:15,538 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/a9de6b9ccfb1433cb5e01a5d8e9924eb as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/a9de6b9ccfb1433cb5e01a5d8e9924eb 2024-12-02T14:19:15,554 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/C of 4eaa960f4ddde5e2082f5ac5eba8cec8 into a9de6b9ccfb1433cb5e01a5d8e9924eb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:15,554 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:15,554 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/C, priority=13, startTime=1733149155358; duration=0sec 2024-12-02T14:19:15,555 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:15,555 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:C 2024-12-02T14:19:15,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/54484cf7983c4b1eae5bfe40ae1c6e59 is 50, key is test_row_0/B:col10/1733149154836/Put/seqid=0 2024-12-02T14:19:15,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149215575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:15,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:15,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:15,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741892_1068 (size=12301) 2024-12-02T14:19:15,612 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/54484cf7983c4b1eae5bfe40ae1c6e59 2024-12-02T14:19:15,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/463a72b549aa4d31b9924c78b3a6b85d is 50, key is test_row_0/C:col10/1733149154836/Put/seqid=0 2024-12-02T14:19:15,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741893_1069 (size=12301) 2024-12-02T14:19:15,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149215689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:15,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T14:19:15,814 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/84b7332e744d4229a9b28fbf34ca4195 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/84b7332e744d4229a9b28fbf34ca4195 2024-12-02T14:19:15,822 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/A of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 84b7332e744d4229a9b28fbf34ca4195(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:15,822 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:15,822 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/A, priority=13, startTime=1733149155356; duration=0sec 2024-12-02T14:19:15,822 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:15,822 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:A 2024-12-02T14:19:15,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149215899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:15,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:16,049 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/463a72b549aa4d31b9924c78b3a6b85d 2024-12-02T14:19:16,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33256 deadline: 1733149216058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:16,060 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 4eaa960f4ddde5e2082f5ac5eba8cec8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:16,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:16,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/daf2c8536a484bdcb5da63128a99e71d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/daf2c8536a484bdcb5da63128a99e71d 2024-12-02T14:19:16,072 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/daf2c8536a484bdcb5da63128a99e71d, entries=150, sequenceid=333, filesize=12.0 K 2024-12-02T14:19:16,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/54484cf7983c4b1eae5bfe40ae1c6e59 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/54484cf7983c4b1eae5bfe40ae1c6e59 2024-12-02T14:19:16,086 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/54484cf7983c4b1eae5bfe40ae1c6e59, entries=150, sequenceid=333, filesize=12.0 K 2024-12-02T14:19:16,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/463a72b549aa4d31b9924c78b3a6b85d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/463a72b549aa4d31b9924c78b3a6b85d 2024-12-02T14:19:16,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33310 deadline: 1733149216089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:16,096 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 4eaa960f4ddde5e2082f5ac5eba8cec8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:16,106 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/463a72b549aa4d31b9924c78b3a6b85d, entries=150, sequenceid=333, filesize=12.0 K 2024-12-02T14:19:16,109 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 641ms, sequenceid=333, compaction requested=false 2024-12-02T14:19:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-02T14:19:16,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-02T14:19:16,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T14:19:16,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:16,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:16,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:16,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:16,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-02T14:19:16,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4370 sec 2024-12-02T14:19:16,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:16,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:16,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.4470 sec 2024-12-02T14:19:16,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/4cc8b721877044699c0ae5f77e7aa0fc is 50, key is test_row_0/A:col10/1733149155549/Put/seqid=0 2024-12-02T14:19:16,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741894_1070 (size=12301) 2024-12-02T14:19:16,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/4cc8b721877044699c0ae5f77e7aa0fc 2024-12-02T14:19:16,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/410671549eac4efd8b42e7f8ea68867c is 50, key is test_row_0/B:col10/1733149155549/Put/seqid=0 2024-12-02T14:19:16,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741895_1071 (size=12301) 2024-12-02T14:19:16,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/410671549eac4efd8b42e7f8ea68867c 2024-12-02T14:19:16,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:16,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149216225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149216225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:16,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/a1210443e48a430ba2ca291fa0a5b10b is 50, key is test_row_0/C:col10/1733149155549/Put/seqid=0 2024-12-02T14:19:16,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741896_1072 (size=12301) 2024-12-02T14:19:16,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/a1210443e48a430ba2ca291fa0a5b10b 2024-12-02T14:19:16,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/4cc8b721877044699c0ae5f77e7aa0fc as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/4cc8b721877044699c0ae5f77e7aa0fc 2024-12-02T14:19:16,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/4cc8b721877044699c0ae5f77e7aa0fc, entries=150, sequenceid=350, filesize=12.0 K 2024-12-02T14:19:16,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/410671549eac4efd8b42e7f8ea68867c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/410671549eac4efd8b42e7f8ea68867c 2024-12-02T14:19:16,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/410671549eac4efd8b42e7f8ea68867c, entries=150, sequenceid=350, filesize=12.0 K 2024-12-02T14:19:16,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/a1210443e48a430ba2ca291fa0a5b10b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/a1210443e48a430ba2ca291fa0a5b10b 2024-12-02T14:19:16,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/a1210443e48a430ba2ca291fa0a5b10b, entries=150, sequenceid=350, filesize=12.0 K 2024-12-02T14:19:16,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 219ms, sequenceid=350, compaction requested=true 2024-12-02T14:19:16,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:16,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:16,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:16,333 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:16,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:16,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:16,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:16,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T14:19:16,333 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:16,336 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:16,336 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:16,336 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/B is initiating minor compaction (all files) 2024-12-02T14:19:16,336 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/A is initiating minor compaction (all files) 2024-12-02T14:19:16,336 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/A in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:16,337 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/84b7332e744d4229a9b28fbf34ca4195, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/daf2c8536a484bdcb5da63128a99e71d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/4cc8b721877044699c0ae5f77e7aa0fc] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=36.6 K 2024-12-02T14:19:16,337 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84b7332e744d4229a9b28fbf34ca4195, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733149154073 2024-12-02T14:19:16,338 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/B in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:16,338 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/bdcba523237944b0aaee6ed1d0bbec85, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/54484cf7983c4b1eae5bfe40ae1c6e59, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/410671549eac4efd8b42e7f8ea68867c] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=36.6 K 2024-12-02T14:19:16,338 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting daf2c8536a484bdcb5da63128a99e71d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733149154754 2024-12-02T14:19:16,338 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting bdcba523237944b0aaee6ed1d0bbec85, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733149154073 2024-12-02T14:19:16,338 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cc8b721877044699c0ae5f77e7aa0fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733149155549 2024-12-02T14:19:16,340 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 54484cf7983c4b1eae5bfe40ae1c6e59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733149154754 2024-12-02T14:19:16,341 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 410671549eac4efd8b42e7f8ea68867c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733149155549 2024-12-02T14:19:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:16,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T14:19:16,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:16,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:16,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:16,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:16,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:16,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:16,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/90d3b907d20f48d2a64e145cd68d46c0 is 50, key is test_row_0/A:col10/1733149156340/Put/seqid=0 2024-12-02T14:19:16,355 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#A#compaction#61 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:16,356 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/d32c3746d01d421abe90911653f3869e is 50, key is test_row_0/A:col10/1733149155549/Put/seqid=0 2024-12-02T14:19:16,358 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#B#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:16,359 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/8e7cc561657b4453832e975c19f3b852 is 50, key is test_row_0/B:col10/1733149155549/Put/seqid=0 2024-12-02T14:19:16,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741897_1073 (size=12301) 2024-12-02T14:19:16,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149216370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:16,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149216371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:16,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/90d3b907d20f48d2a64e145cd68d46c0 2024-12-02T14:19:16,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:16,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741899_1075 (size=12983) 2024-12-02T14:19:16,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741898_1074 (size=12983) 2024-12-02T14:19:16,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/defc3e9abe834e67bc773a7736b445fb is 50, key is test_row_0/B:col10/1733149156340/Put/seqid=0 2024-12-02T14:19:16,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741900_1076 (size=12301) 2024-12-02T14:19:16,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/defc3e9abe834e67bc773a7736b445fb 2024-12-02T14:19:16,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/193f9db2bff340eea13d6d2878ddec0c is 50, key is test_row_0/C:col10/1733149156340/Put/seqid=0 2024-12-02T14:19:16,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741901_1077 (size=12301) 2024-12-02T14:19:16,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/193f9db2bff340eea13d6d2878ddec0c 2024-12-02T14:19:16,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149216478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:16,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/90d3b907d20f48d2a64e145cd68d46c0 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/90d3b907d20f48d2a64e145cd68d46c0 2024-12-02T14:19:16,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149216489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:16,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/90d3b907d20f48d2a64e145cd68d46c0, entries=150, sequenceid=369, filesize=12.0 K 2024-12-02T14:19:16,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/defc3e9abe834e67bc773a7736b445fb as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/defc3e9abe834e67bc773a7736b445fb 2024-12-02T14:19:16,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/defc3e9abe834e67bc773a7736b445fb, entries=150, sequenceid=369, filesize=12.0 K 2024-12-02T14:19:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/193f9db2bff340eea13d6d2878ddec0c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/193f9db2bff340eea13d6d2878ddec0c 2024-12-02T14:19:16,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/193f9db2bff340eea13d6d2878ddec0c, entries=150, sequenceid=369, filesize=12.0 K 2024-12-02T14:19:16,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 179ms, sequenceid=369, compaction requested=true 2024-12-02T14:19:16,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:16,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:A, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:16,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T14:19:16,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:B, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:16,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T14:19:16,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:16,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T14:19:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:16,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T14:19:16,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:16,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:16,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:16,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:16,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:16,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:16,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/75c7a56832324f558b9551dcb232cdbb is 50, key is test_row_0/A:col10/1733149156368/Put/seqid=0 2024-12-02T14:19:16,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741902_1078 (size=12301) 2024-12-02T14:19:16,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/75c7a56832324f558b9551dcb232cdbb 2024-12-02T14:19:16,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/4177529006c9404b92f8455946b6eaec is 50, key is test_row_0/B:col10/1733149156368/Put/seqid=0 2024-12-02T14:19:16,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:16,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149216768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:16,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:16,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:16,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149216770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:16,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:16,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741903_1079 (size=12301) 2024-12-02T14:19:16,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/4177529006c9404b92f8455946b6eaec 2024-12-02T14:19:16,808 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/8e7cc561657b4453832e975c19f3b852 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/8e7cc561657b4453832e975c19f3b852 2024-12-02T14:19:16,818 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/d32c3746d01d421abe90911653f3869e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/d32c3746d01d421abe90911653f3869e 2024-12-02T14:19:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T14:19:16,819 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:16,821 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-02T14:19:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T14:19:16,824 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:16,826 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:16,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:16,829 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/B of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 8e7cc561657b4453832e975c19f3b852(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:16,829 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:16,829 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/B, priority=13, startTime=1733149156333; duration=0sec 2024-12-02T14:19:16,830 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T14:19:16,830 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:B 2024-12-02T14:19:16,830 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-12-02T14:19:16,833 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T14:19:16,834 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T14:19:16,834 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. because compaction request was cancelled 2024-12-02T14:19:16,834 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:A 2024-12-02T14:19:16,834 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:16,835 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/A of 4eaa960f4ddde5e2082f5ac5eba8cec8 into d32c3746d01d421abe90911653f3869e(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:16,835 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:16,835 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/A, priority=13, startTime=1733149156333; duration=0sec 2024-12-02T14:19:16,836 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T14:19:16,836 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:A 2024-12-02T14:19:16,836 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:B 2024-12-02T14:19:16,837 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:16,838 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/C is initiating minor compaction (all files) 2024-12-02T14:19:16,838 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/C in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:16,838 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/a9de6b9ccfb1433cb5e01a5d8e9924eb, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/463a72b549aa4d31b9924c78b3a6b85d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/a1210443e48a430ba2ca291fa0a5b10b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/193f9db2bff340eea13d6d2878ddec0c] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=48.6 K 2024-12-02T14:19:16,838 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-12-02T14:19:16,838 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T14:19:16,838 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T14:19:16,838 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. because compaction request was cancelled 2024-12-02T14:19:16,838 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:C 2024-12-02T14:19:16,839 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T14:19:16,840 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T14:19:16,841 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T14:19:16,841 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. because compaction request was cancelled 2024-12-02T14:19:16,841 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:B 2024-12-02T14:19:16,841 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting a9de6b9ccfb1433cb5e01a5d8e9924eb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733149154073 2024-12-02T14:19:16,842 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 463a72b549aa4d31b9924c78b3a6b85d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733149154754 2024-12-02T14:19:16,842 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting a1210443e48a430ba2ca291fa0a5b10b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733149155549 2024-12-02T14:19:16,843 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 193f9db2bff340eea13d6d2878ddec0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733149156212 2024-12-02T14:19:16,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/14e4a1ecd0764dc7a0ae855fd29aed0b is 50, key is test_row_0/C:col10/1733149156368/Put/seqid=0 2024-12-02T14:19:16,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741904_1080 (size=12301) 2024-12-02T14:19:16,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:16,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149216878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149216879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:16,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:16,883 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:16,884 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/2205925f3809492fb27a2b790618704c is 50, key is test_row_0/C:col10/1733149156340/Put/seqid=0 2024-12-02T14:19:16,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741905_1081 (size=13017) 2024-12-02T14:19:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T14:19:16,936 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/2205925f3809492fb27a2b790618704c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2205925f3809492fb27a2b790618704c 2024-12-02T14:19:16,956 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/C of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 2205925f3809492fb27a2b790618704c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:16,956 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:16,956 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/C, priority=12, startTime=1733149156522; duration=0sec 2024-12-02T14:19:16,956 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:16,956 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:C 2024-12-02T14:19:16,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T14:19:16,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:16,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:16,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:16,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:16,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:17,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149217089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:17,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:17,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149217089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:17,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T14:19:17,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:17,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:17,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:17,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:17,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T14:19:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:17,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/14e4a1ecd0764dc7a0ae855fd29aed0b 2024-12-02T14:19:17,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/75c7a56832324f558b9551dcb232cdbb as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/75c7a56832324f558b9551dcb232cdbb 2024-12-02T14:19:17,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T14:19:17,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:17,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:17,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:17,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:17,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:17,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/75c7a56832324f558b9551dcb232cdbb, entries=150, sequenceid=383, filesize=12.0 K 2024-12-02T14:19:17,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/4177529006c9404b92f8455946b6eaec as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/4177529006c9404b92f8455946b6eaec 2024-12-02T14:19:17,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/4177529006c9404b92f8455946b6eaec, entries=150, sequenceid=383, filesize=12.0 K 2024-12-02T14:19:17,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/14e4a1ecd0764dc7a0ae855fd29aed0b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/14e4a1ecd0764dc7a0ae855fd29aed0b 2024-12-02T14:19:17,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/14e4a1ecd0764dc7a0ae855fd29aed0b, entries=150, sequenceid=383, filesize=12.0 K 2024-12-02T14:19:17,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 632ms, sequenceid=383, compaction requested=true 2024-12-02T14:19:17,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:17,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:17,323 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:17,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:17,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:17,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:17,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:17,323 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:17,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T14:19:17,326 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:17,326 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/A is initiating minor compaction (all files) 2024-12-02T14:19:17,326 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/A in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:17,326 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/d32c3746d01d421abe90911653f3869e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/90d3b907d20f48d2a64e145cd68d46c0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/75c7a56832324f558b9551dcb232cdbb] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=36.7 K 2024-12-02T14:19:17,326 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting d32c3746d01d421abe90911653f3869e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733149155549 2024-12-02T14:19:17,327 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90d3b907d20f48d2a64e145cd68d46c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733149156212 2024-12-02T14:19:17,327 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75c7a56832324f558b9551dcb232cdbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733149156368 2024-12-02T14:19:17,329 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:17,329 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/B is initiating minor compaction (all files) 2024-12-02T14:19:17,329 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/B in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:17,329 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/8e7cc561657b4453832e975c19f3b852, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/defc3e9abe834e67bc773a7736b445fb, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/4177529006c9404b92f8455946b6eaec] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=36.7 K 2024-12-02T14:19:17,331 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e7cc561657b4453832e975c19f3b852, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733149155549 2024-12-02T14:19:17,332 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting defc3e9abe834e67bc773a7736b445fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733149156212 2024-12-02T14:19:17,332 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 4177529006c9404b92f8455946b6eaec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733149156368 2024-12-02T14:19:17,339 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#A#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:17,340 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/b75294e5d21f4ee990397da225021bac is 50, key is test_row_0/A:col10/1733149156368/Put/seqid=0 2024-12-02T14:19:17,347 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:17,348 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/9fd40e5f13944201be20318d2dfc8220 is 50, key is test_row_0/B:col10/1733149156368/Put/seqid=0 2024-12-02T14:19:17,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741906_1082 (size=13085) 2024-12-02T14:19:17,388 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/b75294e5d21f4ee990397da225021bac as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/b75294e5d21f4ee990397da225021bac 2024-12-02T14:19:17,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741907_1083 (size=13085) 2024-12-02T14:19:17,405 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/A of 4eaa960f4ddde5e2082f5ac5eba8cec8 into b75294e5d21f4ee990397da225021bac(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:17,405 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:17,405 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/A, priority=13, startTime=1733149157323; duration=0sec 2024-12-02T14:19:17,405 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:17,405 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:A 2024-12-02T14:19:17,405 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T14:19:17,409 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/9fd40e5f13944201be20318d2dfc8220 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/9fd40e5f13944201be20318d2dfc8220 2024-12-02T14:19:17,409 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T14:19:17,409 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T14:19:17,410 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. because compaction request was cancelled 2024-12-02T14:19:17,410 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:C 2024-12-02T14:19:17,435 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/B of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 9fd40e5f13944201be20318d2dfc8220(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:17,435 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:17,435 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/B, priority=13, startTime=1733149157323; duration=0sec 2024-12-02T14:19:17,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:17,436 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:17,436 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:B 2024-12-02T14:19:17,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-12-02T14:19:17,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149156734, max timestamp=1733149157416Num uniques 859; 2024-12-02T14:19:17,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:19:17,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:17,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:17,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:17,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:17,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:17,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:17,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T14:19:17,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:17,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:17,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:17,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:17,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T14:19:17,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:17,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/3208c6ca895e4655adc5d98b04e2c449 is 50, key is test_row_0/A:col10/1733149157416/Put/seqid=0 2024-12-02T14:19:17,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149217490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149217490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:17,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741908_1084 (size=17181) 2024-12-02T14:19:17,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/3208c6ca895e4655adc5d98b04e2c449 2024-12-02T14:19:17,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/35746747b5d54cb4b5b6516bf631e09e is 50, key is test_row_0/B:col10/1733149157416/Put/seqid=0 2024-12-02T14:19:17,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741909_1085 (size=12301) 2024-12-02T14:19:17,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/35746747b5d54cb4b5b6516bf631e09e 2024-12-02T14:19:17,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/62241b72102a46ea845d2686368ff7d8 is 50, key is test_row_0/C:col10/1733149157416/Put/seqid=0 2024-12-02T14:19:17,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:17,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149217599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:17,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:17,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T14:19:17,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:17,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:17,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:17,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:17,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:17,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149217599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:17,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:17,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:17,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741910_1086 (size=12301) 2024-12-02T14:19:17,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/62241b72102a46ea845d2686368ff7d8 2024-12-02T14:19:17,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/3208c6ca895e4655adc5d98b04e2c449 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/3208c6ca895e4655adc5d98b04e2c449 2024-12-02T14:19:17,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/3208c6ca895e4655adc5d98b04e2c449, entries=250, sequenceid=409, filesize=16.8 K 2024-12-02T14:19:17,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/35746747b5d54cb4b5b6516bf631e09e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/35746747b5d54cb4b5b6516bf631e09e 2024-12-02T14:19:17,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/35746747b5d54cb4b5b6516bf631e09e, entries=150, sequenceid=409, filesize=12.0 K 2024-12-02T14:19:17,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/62241b72102a46ea845d2686368ff7d8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/62241b72102a46ea845d2686368ff7d8 2024-12-02T14:19:17,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/62241b72102a46ea845d2686368ff7d8, entries=150, sequenceid=409, filesize=12.0 K 2024-12-02T14:19:17,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~91.69 KB/93890, heapSize ~267.14 KB/273552, currentSize=87.22 KB/89310 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 216ms, sequenceid=409, compaction requested=true 2024-12-02T14:19:17,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:17,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:17,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:17,653 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T14:19:17,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:17,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:17,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:17,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T14:19:17,653 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:17,654 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T14:19:17,654 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T14:19:17,654 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. because compaction request was cancelled 2024-12-02T14:19:17,654 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:A 2024-12-02T14:19:17,654 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T14:19:17,654 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:17,654 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/C is initiating minor compaction (all files) 2024-12-02T14:19:17,654 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/C in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:17,655 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2205925f3809492fb27a2b790618704c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/14e4a1ecd0764dc7a0ae855fd29aed0b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/62241b72102a46ea845d2686368ff7d8] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=36.7 K 2024-12-02T14:19:17,655 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T14:19:17,655 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T14:19:17,655 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. because compaction request was cancelled 2024-12-02T14:19:17,655 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:B 2024-12-02T14:19:17,655 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 2205925f3809492fb27a2b790618704c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733149156212 2024-12-02T14:19:17,655 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 14e4a1ecd0764dc7a0ae855fd29aed0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733149156368 2024-12-02T14:19:17,656 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 62241b72102a46ea845d2686368ff7d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733149156756 2024-12-02T14:19:17,677 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#C#compaction#74 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:17,678 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/90c6f9e147d249b68128bf1f6baa421f is 50, key is test_row_0/C:col10/1733149157416/Put/seqid=0 2024-12-02T14:19:17,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741911_1087 (size=13119) 2024-12-02T14:19:17,702 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/90c6f9e147d249b68128bf1f6baa421f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/90c6f9e147d249b68128bf1f6baa421f 2024-12-02T14:19:17,712 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/C of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 90c6f9e147d249b68128bf1f6baa421f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:17,712 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:17,712 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/C, priority=13, startTime=1733149157653; duration=0sec 2024-12-02T14:19:17,712 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:17,712 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:C 2024-12-02T14:19:17,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T14:19:17,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:17,760 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T14:19:17,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:17,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:17,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:17,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:17,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:17,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:17,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/e79195bcd8d4486d8c8670db2a69d0ab is 50, key is test_row_0/A:col10/1733149157480/Put/seqid=0 2024-12-02T14:19:17,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741912_1088 (size=12301) 2024-12-02T14:19:17,776 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/e79195bcd8d4486d8c8670db2a69d0ab 2024-12-02T14:19:17,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/91883128ce564355b06bebd19d378b8d is 50, key is test_row_0/B:col10/1733149157480/Put/seqid=0 2024-12-02T14:19:17,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741913_1089 (size=12301) 2024-12-02T14:19:17,802 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/91883128ce564355b06bebd19d378b8d 2024-12-02T14:19:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:17,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:17,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/faa08da2cc15460d8da7f80249724924 is 50, key is test_row_0/C:col10/1733149157480/Put/seqid=0 2024-12-02T14:19:17,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741914_1090 (size=12301) 2024-12-02T14:19:17,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149217839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:17,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149217844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:17,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149217948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149217948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T14:19:18,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149218159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:18,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:18,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149218159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:18,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:18,221 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/faa08da2cc15460d8da7f80249724924 2024-12-02T14:19:18,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/e79195bcd8d4486d8c8670db2a69d0ab as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/e79195bcd8d4486d8c8670db2a69d0ab 2024-12-02T14:19:18,238 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/e79195bcd8d4486d8c8670db2a69d0ab, entries=150, sequenceid=426, filesize=12.0 K 2024-12-02T14:19:18,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/91883128ce564355b06bebd19d378b8d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/91883128ce564355b06bebd19d378b8d 2024-12-02T14:19:18,250 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/91883128ce564355b06bebd19d378b8d, entries=150, sequenceid=426, filesize=12.0 K 2024-12-02T14:19:18,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/faa08da2cc15460d8da7f80249724924 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/faa08da2cc15460d8da7f80249724924 2024-12-02T14:19:18,268 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/faa08da2cc15460d8da7f80249724924, entries=150, sequenceid=426, filesize=12.0 K 2024-12-02T14:19:18,270 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 510ms, sequenceid=426, compaction requested=true 2024-12-02T14:19:18,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:18,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:18,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-02T14:19:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-02T14:19:18,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-02T14:19:18,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4460 sec 2024-12-02T14:19:18,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.4600 sec 2024-12-02T14:19:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:18,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T14:19:18,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:18,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:18,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:18,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:18,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:18,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:18,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/049b71f22c5f4c3b9ce40529e74ba699 is 50, key is test_row_0/A:col10/1733149157836/Put/seqid=0 2024-12-02T14:19:18,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741915_1091 (size=14741) 2024-12-02T14:19:18,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/049b71f22c5f4c3b9ce40529e74ba699 2024-12-02T14:19:18,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149218514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149218516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:18,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/88f04dd7b7d54f0bb53e697d1eee322e is 50, key is test_row_0/B:col10/1733149157836/Put/seqid=0 2024-12-02T14:19:18,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741916_1092 (size=12301) 2024-12-02T14:19:18,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/88f04dd7b7d54f0bb53e697d1eee322e 2024-12-02T14:19:18,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/01d35e26d5354baba7883fe03149ac0a is 50, key is test_row_0/C:col10/1733149157836/Put/seqid=0 2024-12-02T14:19:18,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741917_1093 (size=12301) 2024-12-02T14:19:18,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/01d35e26d5354baba7883fe03149ac0a 2024-12-02T14:19:18,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/049b71f22c5f4c3b9ce40529e74ba699 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/049b71f22c5f4c3b9ce40529e74ba699 2024-12-02T14:19:18,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/049b71f22c5f4c3b9ce40529e74ba699, entries=200, sequenceid=443, filesize=14.4 K 2024-12-02T14:19:18,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/88f04dd7b7d54f0bb53e697d1eee322e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/88f04dd7b7d54f0bb53e697d1eee322e 2024-12-02T14:19:18,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/88f04dd7b7d54f0bb53e697d1eee322e, entries=150, sequenceid=443, filesize=12.0 K 2024-12-02T14:19:18,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/01d35e26d5354baba7883fe03149ac0a as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/01d35e26d5354baba7883fe03149ac0a 2024-12-02T14:19:18,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/01d35e26d5354baba7883fe03149ac0a, entries=150, sequenceid=443, filesize=12.0 K 2024-12-02T14:19:18,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 150ms, sequenceid=443, compaction requested=true 2024-12-02T14:19:18,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:18,625 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:18,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:18,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:18,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:18,627 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57308 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:18,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:18,627 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:18,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:18,627 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/A is initiating minor compaction (all files) 2024-12-02T14:19:18,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:18,627 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/A in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:18,627 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/b75294e5d21f4ee990397da225021bac, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/3208c6ca895e4655adc5d98b04e2c449, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/e79195bcd8d4486d8c8670db2a69d0ab, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/049b71f22c5f4c3b9ce40529e74ba699] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=56.0 K 2024-12-02T14:19:18,628 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting b75294e5d21f4ee990397da225021bac, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733149156368 2024-12-02T14:19:18,628 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3208c6ca895e4655adc5d98b04e2c449, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733149156734 2024-12-02T14:19:18,628 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:18,628 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting e79195bcd8d4486d8c8670db2a69d0ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733149157480 2024-12-02T14:19:18,628 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/B is initiating minor compaction (all files) 2024-12-02T14:19:18,628 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/B in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:18,629 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/9fd40e5f13944201be20318d2dfc8220, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/35746747b5d54cb4b5b6516bf631e09e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/91883128ce564355b06bebd19d378b8d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/88f04dd7b7d54f0bb53e697d1eee322e] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=48.8 K 2024-12-02T14:19:18,629 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 049b71f22c5f4c3b9ce40529e74ba699, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733149157836 2024-12-02T14:19:18,630 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fd40e5f13944201be20318d2dfc8220, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733149156368 2024-12-02T14:19:18,630 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 35746747b5d54cb4b5b6516bf631e09e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733149156756 2024-12-02T14:19:18,631 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 91883128ce564355b06bebd19d378b8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733149157480 2024-12-02T14:19:18,632 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 88f04dd7b7d54f0bb53e697d1eee322e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733149157836 2024-12-02T14:19:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:18,642 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#A#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:18,642 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/8da33d8d63dd4eb7b6d38988896f3ea3 is 50, key is test_row_0/A:col10/1733149157836/Put/seqid=0 2024-12-02T14:19:18,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T14:19:18,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:18,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:18,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:18,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:18,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:18,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:18,649 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#B#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:18,649 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/88c2f88fbfb84e4d8e0235c4336ffb51 is 50, key is test_row_0/B:col10/1733149157836/Put/seqid=0 2024-12-02T14:19:18,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741918_1094 (size=13221) 2024-12-02T14:19:18,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741919_1095 (size=13221) 2024-12-02T14:19:18,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/0f5be3c37b8d4febbdb6f56f67f61928 is 50, key is test_row_0/A:col10/1733149158510/Put/seqid=0 2024-12-02T14:19:18,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149218667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:18,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:18,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:18,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741920_1096 (size=14741) 2024-12-02T14:19:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149218674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:18,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/0f5be3c37b8d4febbdb6f56f67f61928 2024-12-02T14:19:18,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/b78c1f36d516482cb65becc931f11eb3 is 50, key is test_row_0/B:col10/1733149158510/Put/seqid=0 2024-12-02T14:19:18,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741921_1097 (size=12301) 2024-12-02T14:19:18,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/b78c1f36d516482cb65becc931f11eb3 2024-12-02T14:19:18,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/c882676449aa499c9253b166481be1c1 is 50, key is test_row_0/C:col10/1733149158510/Put/seqid=0 2024-12-02T14:19:18,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741922_1098 (size=12301) 2024-12-02T14:19:18,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149218779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:18,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:18,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149218779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T14:19:18,969 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:18,970 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-02T14:19:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T14:19:18,972 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:18,973 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:18,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:18,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149218989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:18,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149218989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:19,067 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/8da33d8d63dd4eb7b6d38988896f3ea3 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/8da33d8d63dd4eb7b6d38988896f3ea3 2024-12-02T14:19:19,074 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/88c2f88fbfb84e4d8e0235c4336ffb51 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/88c2f88fbfb84e4d8e0235c4336ffb51 2024-12-02T14:19:19,078 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/A of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 8da33d8d63dd4eb7b6d38988896f3ea3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:19,079 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T14:19:19,079 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/A, priority=12, startTime=1733149158625; duration=0sec 2024-12-02T14:19:19,079 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:19,079 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:A 2024-12-02T14:19:19,079 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:19,081 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:19,081 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/C is initiating minor compaction (all files) 2024-12-02T14:19:19,081 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/C in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:19,081 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/90c6f9e147d249b68128bf1f6baa421f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/faa08da2cc15460d8da7f80249724924, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/01d35e26d5354baba7883fe03149ac0a] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=36.8 K 2024-12-02T14:19:19,082 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90c6f9e147d249b68128bf1f6baa421f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733149156756 2024-12-02T14:19:19,082 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting faa08da2cc15460d8da7f80249724924, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733149157480 2024-12-02T14:19:19,082 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01d35e26d5354baba7883fe03149ac0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733149157836 2024-12-02T14:19:19,087 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/B of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 88c2f88fbfb84e4d8e0235c4336ffb51(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:19,087 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:19,087 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/B, priority=12, startTime=1733149158627; duration=0sec 2024-12-02T14:19:19,088 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:19,088 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:B 2024-12-02T14:19:19,098 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#C#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:19,098 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/f622ae2c72cf4a929ff236140d6c96ef is 50, key is test_row_0/C:col10/1733149157836/Put/seqid=0 2024-12-02T14:19:19,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741923_1099 (size=13221) 2024-12-02T14:19:19,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T14:19:19,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:19,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:19,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:19,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:19,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:19,129 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/f622ae2c72cf4a929ff236140d6c96ef as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/f622ae2c72cf4a929ff236140d6c96ef 2024-12-02T14:19:19,143 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/C of 4eaa960f4ddde5e2082f5ac5eba8cec8 into f622ae2c72cf4a929ff236140d6c96ef(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:19,143 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:19,143 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/C, priority=13, startTime=1733149158627; duration=0sec 2024-12-02T14:19:19,143 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:19,144 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:C 2024-12-02T14:19:19,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/c882676449aa499c9253b166481be1c1 2024-12-02T14:19:19,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/0f5be3c37b8d4febbdb6f56f67f61928 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/0f5be3c37b8d4febbdb6f56f67f61928 2024-12-02T14:19:19,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/0f5be3c37b8d4febbdb6f56f67f61928, entries=200, sequenceid=462, filesize=14.4 K 2024-12-02T14:19:19,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/b78c1f36d516482cb65becc931f11eb3 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b78c1f36d516482cb65becc931f11eb3 2024-12-02T14:19:19,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b78c1f36d516482cb65becc931f11eb3, entries=150, sequenceid=462, filesize=12.0 K 2024-12-02T14:19:19,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/c882676449aa499c9253b166481be1c1 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/c882676449aa499c9253b166481be1c1 2024-12-02T14:19:19,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/c882676449aa499c9253b166481be1c1, entries=150, sequenceid=462, filesize=12.0 K 2024-12-02T14:19:19,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 569ms, sequenceid=462, compaction requested=false 2024-12-02T14:19:19,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:19,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T14:19:19,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:19,281 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T14:19:19,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:19,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:19,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:19,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:19,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:19,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T14:19:19,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/c8adcf6419d5497ca1dee1b1e39e5567 is 50, key is test_row_0/A:col10/1733149158659/Put/seqid=0 2024-12-02T14:19:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:19,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:19,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741924_1100 (size=12301) 2024-12-02T14:19:19,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149219345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:19,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:19,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:19,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:19,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149219348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:19,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:19,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:19,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:19,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149219459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:19,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149219460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T14:19:19,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149219669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:19,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:19,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149219669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:19,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:19,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:19,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:19,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:19,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:19,728 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/c8adcf6419d5497ca1dee1b1e39e5567 2024-12-02T14:19:19,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/6d3e96cf90d24b76a0421afb87d069eb is 50, key is test_row_0/B:col10/1733149158659/Put/seqid=0 2024-12-02T14:19:19,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741925_1101 (size=12301) 2024-12-02T14:19:19,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149219978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:19,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149219978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:19,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:19,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:19,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T14:19:20,143 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/6d3e96cf90d24b76a0421afb87d069eb 2024-12-02T14:19:20,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/7469a00a984042d7b71994ae2323a8ec is 50, key is test_row_0/C:col10/1733149158659/Put/seqid=0 2024-12-02T14:19:20,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741926_1102 (size=12301) 2024-12-02T14:19:20,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149220489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:20,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149220489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:20,560 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/7469a00a984042d7b71994ae2323a8ec 2024-12-02T14:19:20,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/c8adcf6419d5497ca1dee1b1e39e5567 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c8adcf6419d5497ca1dee1b1e39e5567 2024-12-02T14:19:20,577 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c8adcf6419d5497ca1dee1b1e39e5567, entries=150, sequenceid=478, filesize=12.0 K 2024-12-02T14:19:20,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/6d3e96cf90d24b76a0421afb87d069eb as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/6d3e96cf90d24b76a0421afb87d069eb 2024-12-02T14:19:20,604 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/6d3e96cf90d24b76a0421afb87d069eb, entries=150, sequenceid=478, filesize=12.0 K 2024-12-02T14:19:20,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/7469a00a984042d7b71994ae2323a8ec as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/7469a00a984042d7b71994ae2323a8ec 2024-12-02T14:19:20,621 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/7469a00a984042d7b71994ae2323a8ec, entries=150, sequenceid=478, filesize=12.0 K 2024-12-02T14:19:20,623 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 1342ms, sequenceid=478, compaction requested=true 2024-12-02T14:19:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-02T14:19:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-02T14:19:20,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-02T14:19:20,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6520 sec 2024-12-02T14:19:20,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.6620 sec 2024-12-02T14:19:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T14:19:21,130 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:21,131 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-02T14:19:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T14:19:21,135 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:21,137 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:21,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T14:19:21,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T14:19:21,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:21,294 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T14:19:21,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:21,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:21,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:21,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:21,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:21,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:21,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/36ee31d9f41f4524b05dbd8041163c89 is 50, key is test_row_0/A:col10/1733149159343/Put/seqid=0 2024-12-02T14:19:21,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741927_1103 (size=12301) 2024-12-02T14:19:21,338 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/36ee31d9f41f4524b05dbd8041163c89 2024-12-02T14:19:21,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/26d2a5f2a4e64d288971559098d9c008 is 50, key is test_row_0/B:col10/1733149159343/Put/seqid=0 2024-12-02T14:19:21,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741928_1104 (size=12301) 2024-12-02T14:19:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T14:19:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:21,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:21,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149221525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:21,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:21,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:21,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:21,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149221527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:21,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149221639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:21,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149221639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T14:19:21,780 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/26d2a5f2a4e64d288971559098d9c008 2024-12-02T14:19:21,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/9a1145dcb1fd41eea3d85b5262a6cf86 is 50, key is test_row_0/C:col10/1733149159343/Put/seqid=0 2024-12-02T14:19:21,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741929_1105 (size=12301) 2024-12-02T14:19:21,809 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/9a1145dcb1fd41eea3d85b5262a6cf86 2024-12-02T14:19:21,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/36ee31d9f41f4524b05dbd8041163c89 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/36ee31d9f41f4524b05dbd8041163c89 2024-12-02T14:19:21,824 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/36ee31d9f41f4524b05dbd8041163c89, entries=150, sequenceid=497, filesize=12.0 K 2024-12-02T14:19:21,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/26d2a5f2a4e64d288971559098d9c008 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/26d2a5f2a4e64d288971559098d9c008 2024-12-02T14:19:21,838 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/26d2a5f2a4e64d288971559098d9c008, entries=150, sequenceid=497, filesize=12.0 K 2024-12-02T14:19:21,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/9a1145dcb1fd41eea3d85b5262a6cf86 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/9a1145dcb1fd41eea3d85b5262a6cf86 2024-12-02T14:19:21,848 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/9a1145dcb1fd41eea3d85b5262a6cf86, entries=150, sequenceid=497, filesize=12.0 K 2024-12-02T14:19:21,849 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 555ms, sequenceid=497, compaction requested=true 2024-12-02T14:19:21,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:21,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:21,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-02T14:19:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-02T14:19:21,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T14:19:21,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:21,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:21,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:21,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:21,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:21,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:21,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-02T14:19:21,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 714 msec 2024-12-02T14:19:21,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/fddaa248b2c54423b2fb1d323a307984 is 50, key is test_row_0/A:col10/1733149161523/Put/seqid=0 2024-12-02T14:19:21,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 729 msec 2024-12-02T14:19:21,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741930_1106 (size=14741) 2024-12-02T14:19:21,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/fddaa248b2c54423b2fb1d323a307984 2024-12-02T14:19:21,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/9785bff3af0b433a902be14aebb61ac8 is 50, key is test_row_0/B:col10/1733149161523/Put/seqid=0 2024-12-02T14:19:21,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741931_1107 (size=12301) 2024-12-02T14:19:21,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149221905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:21,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:21,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:21,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:21,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149221908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:22,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149222018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149222018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:22,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149222228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:22,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:22,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149222228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T14:19:22,278 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:22,279 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-02T14:19:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T14:19:22,282 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:22,282 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:22,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:22,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/9785bff3af0b433a902be14aebb61ac8 2024-12-02T14:19:22,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/b5155ded76cc43c1bacd721bfcc96acf is 50, key is test_row_0/C:col10/1733149161523/Put/seqid=0 2024-12-02T14:19:22,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741932_1108 (size=12301) 2024-12-02T14:19:22,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/b5155ded76cc43c1bacd721bfcc96acf 2024-12-02T14:19:22,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/fddaa248b2c54423b2fb1d323a307984 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/fddaa248b2c54423b2fb1d323a307984 2024-12-02T14:19:22,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/fddaa248b2c54423b2fb1d323a307984, entries=200, sequenceid=511, filesize=14.4 K 2024-12-02T14:19:22,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/9785bff3af0b433a902be14aebb61ac8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/9785bff3af0b433a902be14aebb61ac8 2024-12-02T14:19:22,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/9785bff3af0b433a902be14aebb61ac8, entries=150, sequenceid=511, filesize=12.0 K 2024-12-02T14:19:22,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/b5155ded76cc43c1bacd721bfcc96acf as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b5155ded76cc43c1bacd721bfcc96acf 2024-12-02T14:19:22,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b5155ded76cc43c1bacd721bfcc96acf, entries=150, sequenceid=511, filesize=12.0 K 2024-12-02T14:19:22,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 486ms, sequenceid=511, compaction requested=true 2024-12-02T14:19:22,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:22,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:22,337 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T14:19:22,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:22,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:22,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:22,337 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T14:19:22,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:22,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:22,338 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62425 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T14:19:22,338 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 67305 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T14:19:22,339 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/A is initiating minor compaction (all files) 2024-12-02T14:19:22,338 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/B is initiating minor compaction (all files) 2024-12-02T14:19:22,339 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/A in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:22,339 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/B in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:22,339 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/8da33d8d63dd4eb7b6d38988896f3ea3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/0f5be3c37b8d4febbdb6f56f67f61928, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c8adcf6419d5497ca1dee1b1e39e5567, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/36ee31d9f41f4524b05dbd8041163c89, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/fddaa248b2c54423b2fb1d323a307984] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=65.7 K 2024-12-02T14:19:22,339 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/88c2f88fbfb84e4d8e0235c4336ffb51, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b78c1f36d516482cb65becc931f11eb3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/6d3e96cf90d24b76a0421afb87d069eb, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/26d2a5f2a4e64d288971559098d9c008, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/9785bff3af0b433a902be14aebb61ac8] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=61.0 K 2024-12-02T14:19:22,339 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 88c2f88fbfb84e4d8e0235c4336ffb51, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733149157836 2024-12-02T14:19:22,339 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8da33d8d63dd4eb7b6d38988896f3ea3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733149157836 2024-12-02T14:19:22,340 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f5be3c37b8d4febbdb6f56f67f61928, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733149158497 2024-12-02T14:19:22,340 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting b78c1f36d516482cb65becc931f11eb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733149158497 2024-12-02T14:19:22,341 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8adcf6419d5497ca1dee1b1e39e5567, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1733149158647 2024-12-02T14:19:22,341 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d3e96cf90d24b76a0421afb87d069eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1733149158647 2024-12-02T14:19:22,341 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 26d2a5f2a4e64d288971559098d9c008, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733149159337 2024-12-02T14:19:22,341 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36ee31d9f41f4524b05dbd8041163c89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733149159337 2024-12-02T14:19:22,341 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting fddaa248b2c54423b2fb1d323a307984, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1733149161510 2024-12-02T14:19:22,341 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 9785bff3af0b433a902be14aebb61ac8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1733149161523 2024-12-02T14:19:22,351 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#A#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:22,351 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#B#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:22,352 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/8f6ab76ebb534398aee88e56e58f56f9 is 50, key is test_row_0/B:col10/1733149161523/Put/seqid=0 2024-12-02T14:19:22,352 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/d2acbcd486a54a758ce336acd9a47d3b is 50, key is test_row_0/A:col10/1733149161523/Put/seqid=0 2024-12-02T14:19:22,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741934_1110 (size=13391) 2024-12-02T14:19:22,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741933_1109 (size=13391) 2024-12-02T14:19:22,366 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/d2acbcd486a54a758ce336acd9a47d3b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/d2acbcd486a54a758ce336acd9a47d3b 2024-12-02T14:19:22,373 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/A of 4eaa960f4ddde5e2082f5ac5eba8cec8 into d2acbcd486a54a758ce336acd9a47d3b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:22,373 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:22,373 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/A, priority=11, startTime=1733149162337; duration=0sec 2024-12-02T14:19:22,373 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:22,373 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:A 2024-12-02T14:19:22,373 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T14:19:22,375 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62425 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T14:19:22,375 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/C is initiating minor compaction (all files) 2024-12-02T14:19:22,375 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/C in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:22,375 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/f622ae2c72cf4a929ff236140d6c96ef, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/c882676449aa499c9253b166481be1c1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/7469a00a984042d7b71994ae2323a8ec, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/9a1145dcb1fd41eea3d85b5262a6cf86, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b5155ded76cc43c1bacd721bfcc96acf] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=61.0 K 2024-12-02T14:19:22,376 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting f622ae2c72cf4a929ff236140d6c96ef, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733149157836 2024-12-02T14:19:22,376 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting c882676449aa499c9253b166481be1c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733149158497 2024-12-02T14:19:22,376 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7469a00a984042d7b71994ae2323a8ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1733149158647 2024-12-02T14:19:22,377 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a1145dcb1fd41eea3d85b5262a6cf86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733149159337 2024-12-02T14:19:22,377 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5155ded76cc43c1bacd721bfcc96acf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1733149161523 2024-12-02T14:19:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T14:19:22,388 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#C#compaction#98 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:22,389 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/b8d02a77f27b4109bb93537b577566a2 is 50, key is test_row_0/C:col10/1733149161523/Put/seqid=0 2024-12-02T14:19:22,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741935_1111 (size=13391) 2024-12-02T14:19:22,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T14:19:22,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:22,436 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T14:19:22,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:22,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:22,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:22,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:22,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:22,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:22,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/bc39b0fc210c43a2a311c371f388fdd0 is 50, key is test_row_0/A:col10/1733149161904/Put/seqid=0 2024-12-02T14:19:22,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741936_1112 (size=12301) 2024-12-02T14:19:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:22,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. as already flushing 2024-12-02T14:19:22,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149222560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:22,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149222561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T14:19:22,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149222668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:22,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:22,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149222669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:22,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:22,763 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/8f6ab76ebb534398aee88e56e58f56f9 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/8f6ab76ebb534398aee88e56e58f56f9 2024-12-02T14:19:22,769 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/B of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 8f6ab76ebb534398aee88e56e58f56f9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:22,769 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:22,769 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/B, priority=11, startTime=1733149162337; duration=0sec 2024-12-02T14:19:22,769 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:22,769 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:B 2024-12-02T14:19:22,799 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/b8d02a77f27b4109bb93537b577566a2 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b8d02a77f27b4109bb93537b577566a2 2024-12-02T14:19:22,805 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/C of 4eaa960f4ddde5e2082f5ac5eba8cec8 into b8d02a77f27b4109bb93537b577566a2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:22,805 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:22,805 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/C, priority=11, startTime=1733149162337; duration=0sec 2024-12-02T14:19:22,805 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:22,805 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:C 2024-12-02T14:19:22,845 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/bc39b0fc210c43a2a311c371f388fdd0 2024-12-02T14:19:22,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/b5bab1e6242c4c78a634edf3b807a038 is 50, key is test_row_0/B:col10/1733149161904/Put/seqid=0 2024-12-02T14:19:22,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741937_1113 (size=12301) 2024-12-02T14:19:22,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149222878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:22,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149222878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:22,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:22,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T14:19:23,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149223188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:23,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149223188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:23,259 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/b5bab1e6242c4c78a634edf3b807a038 2024-12-02T14:19:23,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/2ab3bf97718546ea9dc7db7db4123ff7 is 50, key is test_row_0/C:col10/1733149161904/Put/seqid=0 2024-12-02T14:19:23,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741938_1114 (size=12301) 2024-12-02T14:19:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T14:19:23,675 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/2ab3bf97718546ea9dc7db7db4123ff7 2024-12-02T14:19:23,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/bc39b0fc210c43a2a311c371f388fdd0 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/bc39b0fc210c43a2a311c371f388fdd0 2024-12-02T14:19:23,687 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/bc39b0fc210c43a2a311c371f388fdd0, entries=150, sequenceid=531, filesize=12.0 K 2024-12-02T14:19:23,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/b5bab1e6242c4c78a634edf3b807a038 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b5bab1e6242c4c78a634edf3b807a038 2024-12-02T14:19:23,694 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b5bab1e6242c4c78a634edf3b807a038, entries=150, sequenceid=531, filesize=12.0 K 2024-12-02T14:19:23,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/2ab3bf97718546ea9dc7db7db4123ff7 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2ab3bf97718546ea9dc7db7db4123ff7 2024-12-02T14:19:23,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33276 deadline: 1733149223698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:23,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33264 deadline: 1733149223698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:23,702 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2ab3bf97718546ea9dc7db7db4123ff7, entries=150, sequenceid=531, filesize=12.0 K 2024-12-02T14:19:23,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:23,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:23,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:23,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:23,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:23,702 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 1267ms, sequenceid=531, compaction requested=false 2024-12-02T14:19:23,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:19:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-02T14:19:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-02T14:19:23,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-02T14:19:23,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4210 sec 2024-12-02T14:19:23,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.4300 sec 2024-12-02T14:19:23,899 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-02T14:19:23,899 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:23,899 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-02T14:19:23,899 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-02T14:19:23,899 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:23,899 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:23,900 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-02T14:19:23,900 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:23,903 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:23,903 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:23,903 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:23,903 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:23,904 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:23,904 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:23,904 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:23,904 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:23,904 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:23,904 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:23,904 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:23,904 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:24,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T14:19:24,293 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-02T14:19:24,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:24,294 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:24,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:24,294 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:24,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:24,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:24,294 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:24,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:24,294 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:24,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:24,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/03565537cc5e40f38e4d2075f91c16b5 is 50, key is test_row_0/A:col10/1733149162553/Put/seqid=0 2024-12-02T14:19:24,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741939_1115 (size=12301) 2024-12-02T14:19:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T14:19:24,429 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:24,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/03565537cc5e40f38e4d2075f91c16b5 2024-12-02T14:19:24,721 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-02T14:19:24,721 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-02T14:19:24,721 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:24,721 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:24,721 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:24,721 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:24,721 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:24,721 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:24,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:24,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:24,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/f15dd530d29f4ab0804339d208ef1c03 is 50, key is test_row_0/B:col10/1733149162553/Put/seqid=0 2024-12-02T14:19:24,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741940_1116 (size=12301) 2024-12-02T14:19:25,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/f15dd530d29f4ab0804339d208ef1c03 2024-12-02T14:19:25,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/b7fdc9058a5f4d87b21ef9807c32c0b8 is 50, key is test_row_0/C:col10/1733149162553/Put/seqid=0 2024-12-02T14:19:25,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741941_1117 (size=12301) 2024-12-02T14:19:25,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/b7fdc9058a5f4d87b21ef9807c32c0b8 2024-12-02T14:19:25,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/03565537cc5e40f38e4d2075f91c16b5 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/03565537cc5e40f38e4d2075f91c16b5 2024-12-02T14:19:25,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/03565537cc5e40f38e4d2075f91c16b5, entries=150, sequenceid=547, filesize=12.0 K 2024-12-02T14:19:25,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/f15dd530d29f4ab0804339d208ef1c03 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/f15dd530d29f4ab0804339d208ef1c03 2024-12-02T14:19:25,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/f15dd530d29f4ab0804339d208ef1c03, entries=150, sequenceid=547, filesize=12.0 K 2024-12-02T14:19:25,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/b7fdc9058a5f4d87b21ef9807c32c0b8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b7fdc9058a5f4d87b21ef9807c32c0b8 2024-12-02T14:19:25,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b7fdc9058a5f4d87b21ef9807c32c0b8, entries=150, sequenceid=547, filesize=12.0 K 2024-12-02T14:19:25,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=13.42 KB/13740 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 1299ms, sequenceid=547, compaction requested=true 2024-12-02T14:19:25,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:25,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:25,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:25,593 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:25,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:25,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:25,593 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:25,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4eaa960f4ddde5e2082f5ac5eba8cec8:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:25,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:25,594 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:25,594 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:25,594 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/A is initiating minor compaction (all files) 2024-12-02T14:19:25,594 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/B is initiating minor compaction (all files) 2024-12-02T14:19:25,594 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/A in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:25,594 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/B in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:25,594 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/d2acbcd486a54a758ce336acd9a47d3b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/bc39b0fc210c43a2a311c371f388fdd0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/03565537cc5e40f38e4d2075f91c16b5] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=37.1 K 2024-12-02T14:19:25,594 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/8f6ab76ebb534398aee88e56e58f56f9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b5bab1e6242c4c78a634edf3b807a038, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/f15dd530d29f4ab0804339d208ef1c03] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=37.1 K 2024-12-02T14:19:25,595 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2acbcd486a54a758ce336acd9a47d3b, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1733149161523 2024-12-02T14:19:25,595 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f6ab76ebb534398aee88e56e58f56f9, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1733149161523 2024-12-02T14:19:25,595 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc39b0fc210c43a2a311c371f388fdd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733149161888 2024-12-02T14:19:25,595 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting b5bab1e6242c4c78a634edf3b807a038, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733149161888 2024-12-02T14:19:25,595 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03565537cc5e40f38e4d2075f91c16b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733149162553 2024-12-02T14:19:25,595 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting f15dd530d29f4ab0804339d208ef1c03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733149162553 2024-12-02T14:19:25,603 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#A#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:25,603 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#B#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:25,604 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/c3d608fb582b48199ee84d57797edfce is 50, key is test_row_0/A:col10/1733149162553/Put/seqid=0 2024-12-02T14:19:25,604 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/88061035ccfa435a9249780bd4fafd7e is 50, key is test_row_0/B:col10/1733149162553/Put/seqid=0 2024-12-02T14:19:25,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741943_1119 (size=13493) 2024-12-02T14:19:25,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741942_1118 (size=13493) 2024-12-02T14:19:26,022 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/c3d608fb582b48199ee84d57797edfce as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c3d608fb582b48199ee84d57797edfce 2024-12-02T14:19:26,022 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/88061035ccfa435a9249780bd4fafd7e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/88061035ccfa435a9249780bd4fafd7e 2024-12-02T14:19:26,027 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/A of 4eaa960f4ddde5e2082f5ac5eba8cec8 into c3d608fb582b48199ee84d57797edfce(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:26,027 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/B of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 88061035ccfa435a9249780bd4fafd7e(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:26,028 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:26,028 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:26,028 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/A, priority=13, startTime=1733149165593; duration=0sec 2024-12-02T14:19:26,028 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/B, priority=13, startTime=1733149165593; duration=0sec 2024-12-02T14:19:26,028 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:26,028 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:26,028 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:A 2024-12-02T14:19:26,028 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:B 2024-12-02T14:19:26,028 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:26,029 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:26,029 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 4eaa960f4ddde5e2082f5ac5eba8cec8/C is initiating minor compaction (all files) 2024-12-02T14:19:26,029 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4eaa960f4ddde5e2082f5ac5eba8cec8/C in TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:26,029 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b8d02a77f27b4109bb93537b577566a2, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2ab3bf97718546ea9dc7db7db4123ff7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b7fdc9058a5f4d87b21ef9807c32c0b8] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp, totalSize=37.1 K 2024-12-02T14:19:26,030 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8d02a77f27b4109bb93537b577566a2, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1733149161523 2024-12-02T14:19:26,030 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ab3bf97718546ea9dc7db7db4123ff7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733149161888 2024-12-02T14:19:26,030 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7fdc9058a5f4d87b21ef9807c32c0b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733149162553 2024-12-02T14:19:26,037 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4eaa960f4ddde5e2082f5ac5eba8cec8#C#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:26,037 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/6af8276460a04405ae4d316ebc49e5d5 is 50, key is test_row_0/C:col10/1733149162553/Put/seqid=0 2024-12-02T14:19:26,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741944_1120 (size=13493) 2024-12-02T14:19:26,132 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-02T14:19:26,132 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:26,132 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:26,133 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:26,133 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:26,181 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-02T14:19:26,181 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:26,181 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:26,181 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:26,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:26,181 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T14:19:26,182 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-12-02T14:19:26,182 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 229 2024-12-02T14:19:26,182 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-12-02T14:19:26,182 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-12-02T14:19:26,182 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-12-02T14:19:26,182 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T14:19:26,182 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2880 2024-12-02T14:19:26,182 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2962 2024-12-02T14:19:26,182 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T14:19:26,182 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1341 2024-12-02T14:19:26,182 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4023 rows 2024-12-02T14:19:26,182 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1336 2024-12-02T14:19:26,182 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4003 rows 2024-12-02T14:19:26,182 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T14:19:26,183 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T14:19:26,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:26,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:26,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:26,191 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T14:19:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T14:19:26,200 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149166199"}]},"ts":"1733149166199"} 2024-12-02T14:19:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T14:19:26,202 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T14:19:26,202 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T14:19:26,204 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T14:19:26,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4eaa960f4ddde5e2082f5ac5eba8cec8, UNASSIGN}] 2024-12-02T14:19:26,208 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4eaa960f4ddde5e2082f5ac5eba8cec8, UNASSIGN 2024-12-02T14:19:26,210 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=4eaa960f4ddde5e2082f5ac5eba8cec8, regionState=CLOSING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:26,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4eaa960f4ddde5e2082f5ac5eba8cec8, UNASSIGN because future has completed 2024-12-02T14:19:26,212 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T14:19:26,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:19:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T14:19:26,372 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:26,373 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T14:19:26,374 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 4eaa960f4ddde5e2082f5ac5eba8cec8, disabling compactions & flushes 2024-12-02T14:19:26,375 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:26,446 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/6af8276460a04405ae4d316ebc49e5d5 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/6af8276460a04405ae4d316ebc49e5d5 2024-12-02T14:19:26,453 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4eaa960f4ddde5e2082f5ac5eba8cec8/C of 4eaa960f4ddde5e2082f5ac5eba8cec8 into 6af8276460a04405ae4d316ebc49e5d5(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:26,453 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: 2024-12-02T14:19:26,453 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., storeName=4eaa960f4ddde5e2082f5ac5eba8cec8/C, priority=13, startTime=1733149165593; duration=0sec 2024-12-02T14:19:26,453 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:26,453 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:26,453 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4eaa960f4ddde5e2082f5ac5eba8cec8:C 2024-12-02T14:19:26,453 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:26,453 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. after waiting 0 ms 2024-12-02T14:19:26,454 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:26,454 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-02T14:19:26,454 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=A 2024-12-02T14:19:26,454 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:26,454 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=B 2024-12-02T14:19:26,454 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:26,454 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4eaa960f4ddde5e2082f5ac5eba8cec8, store=C 2024-12-02T14:19:26,454 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:26,458 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/3bdfa7fb8b4f42b89167a48b31e032e0 is 50, key is test_row_0/A:col10/1733149166179/Put/seqid=0 2024-12-02T14:19:26,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741945_1121 (size=9857) 2024-12-02T14:19:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T14:19:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T14:19:26,863 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/3bdfa7fb8b4f42b89167a48b31e032e0 2024-12-02T14:19:26,877 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/b6b00622595f4f86aafb5713c2074418 is 50, key is test_row_0/B:col10/1733149166179/Put/seqid=0 2024-12-02T14:19:26,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741946_1122 (size=9857) 2024-12-02T14:19:27,047 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T14:19:27,281 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/b6b00622595f4f86aafb5713c2074418 2024-12-02T14:19:27,289 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/a420c9aaef1843a9b26d75b68fdc1a05 is 50, key is test_row_0/C:col10/1733149166179/Put/seqid=0 2024-12-02T14:19:27,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741947_1123 (size=9857) 2024-12-02T14:19:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T14:19:27,694 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/a420c9aaef1843a9b26d75b68fdc1a05 2024-12-02T14:19:27,699 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/A/3bdfa7fb8b4f42b89167a48b31e032e0 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/3bdfa7fb8b4f42b89167a48b31e032e0 2024-12-02T14:19:27,704 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/3bdfa7fb8b4f42b89167a48b31e032e0, entries=100, sequenceid=557, filesize=9.6 K 2024-12-02T14:19:27,705 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/B/b6b00622595f4f86aafb5713c2074418 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b6b00622595f4f86aafb5713c2074418 2024-12-02T14:19:27,709 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b6b00622595f4f86aafb5713c2074418, entries=100, sequenceid=557, filesize=9.6 K 2024-12-02T14:19:27,712 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/.tmp/C/a420c9aaef1843a9b26d75b68fdc1a05 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/a420c9aaef1843a9b26d75b68fdc1a05 2024-12-02T14:19:27,717 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/a420c9aaef1843a9b26d75b68fdc1a05, entries=100, sequenceid=557, filesize=9.6 K 2024-12-02T14:19:27,718 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 1264ms, sequenceid=557, compaction requested=false 2024-12-02T14:19:27,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/521c3ccdbc4e4f0281911a19498c77dd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c0c2f547548948ccbb64ec7fe84ef1dc, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/3800a9e5b40945c694ca7a8506f240d9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/6db3ce214d214a3fa5be306ab019e64c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ab88c1ff9b8548ec950ed50763449745, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/64693bce5986433a982d13b6af03ff6b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/54c01548ad7541fe8793d662cafeb648, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/2b438b43cb81435584ef8b1990d1f31b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/b7479d9dec994fcb8e17bdfcccb09a13, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/437af98b938743b488cb8913454f2d15, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/7d31eef675d3403388480bc9662e8dfb, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ebf614b3659847c1a775cc1e91201053, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c1185de9bfd34d29a88ca7063512e40b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/b875a627322746fc94b4a007515a2ae8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/63f6afa2d0534ec9bf593f6114e13b22, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ffd108b0695540c895db489a2b443256, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/615f00dba520419c96a253db025ca83c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/84b7332e744d4229a9b28fbf34ca4195, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/daf2c8536a484bdcb5da63128a99e71d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/d32c3746d01d421abe90911653f3869e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/4cc8b721877044699c0ae5f77e7aa0fc, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/90d3b907d20f48d2a64e145cd68d46c0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/b75294e5d21f4ee990397da225021bac, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/75c7a56832324f558b9551dcb232cdbb, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/3208c6ca895e4655adc5d98b04e2c449, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/e79195bcd8d4486d8c8670db2a69d0ab, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/049b71f22c5f4c3b9ce40529e74ba699, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/8da33d8d63dd4eb7b6d38988896f3ea3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/0f5be3c37b8d4febbdb6f56f67f61928, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c8adcf6419d5497ca1dee1b1e39e5567, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/36ee31d9f41f4524b05dbd8041163c89, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/fddaa248b2c54423b2fb1d323a307984, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/d2acbcd486a54a758ce336acd9a47d3b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/bc39b0fc210c43a2a311c371f388fdd0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/03565537cc5e40f38e4d2075f91c16b5] to archive 2024-12-02T14:19:27,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:19:27,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/521c3ccdbc4e4f0281911a19498c77dd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/521c3ccdbc4e4f0281911a19498c77dd 2024-12-02T14:19:27,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c0c2f547548948ccbb64ec7fe84ef1dc to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c0c2f547548948ccbb64ec7fe84ef1dc 2024-12-02T14:19:27,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/3800a9e5b40945c694ca7a8506f240d9 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/3800a9e5b40945c694ca7a8506f240d9 2024-12-02T14:19:27,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/6db3ce214d214a3fa5be306ab019e64c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/6db3ce214d214a3fa5be306ab019e64c 2024-12-02T14:19:27,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ab88c1ff9b8548ec950ed50763449745 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ab88c1ff9b8548ec950ed50763449745 2024-12-02T14:19:27,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/64693bce5986433a982d13b6af03ff6b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/64693bce5986433a982d13b6af03ff6b 2024-12-02T14:19:27,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/54c01548ad7541fe8793d662cafeb648 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/54c01548ad7541fe8793d662cafeb648 2024-12-02T14:19:27,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/2b438b43cb81435584ef8b1990d1f31b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/2b438b43cb81435584ef8b1990d1f31b 2024-12-02T14:19:27,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/b7479d9dec994fcb8e17bdfcccb09a13 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/b7479d9dec994fcb8e17bdfcccb09a13 2024-12-02T14:19:27,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/437af98b938743b488cb8913454f2d15 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/437af98b938743b488cb8913454f2d15 2024-12-02T14:19:27,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/7d31eef675d3403388480bc9662e8dfb to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/7d31eef675d3403388480bc9662e8dfb 2024-12-02T14:19:27,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ebf614b3659847c1a775cc1e91201053 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ebf614b3659847c1a775cc1e91201053 2024-12-02T14:19:27,740 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c1185de9bfd34d29a88ca7063512e40b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c1185de9bfd34d29a88ca7063512e40b 2024-12-02T14:19:27,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/b875a627322746fc94b4a007515a2ae8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/b875a627322746fc94b4a007515a2ae8 2024-12-02T14:19:27,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/63f6afa2d0534ec9bf593f6114e13b22 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/63f6afa2d0534ec9bf593f6114e13b22 2024-12-02T14:19:27,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ffd108b0695540c895db489a2b443256 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/ffd108b0695540c895db489a2b443256 2024-12-02T14:19:27,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/615f00dba520419c96a253db025ca83c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/615f00dba520419c96a253db025ca83c 2024-12-02T14:19:27,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/84b7332e744d4229a9b28fbf34ca4195 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/84b7332e744d4229a9b28fbf34ca4195 2024-12-02T14:19:27,748 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/daf2c8536a484bdcb5da63128a99e71d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/daf2c8536a484bdcb5da63128a99e71d 2024-12-02T14:19:27,749 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/d32c3746d01d421abe90911653f3869e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/d32c3746d01d421abe90911653f3869e 2024-12-02T14:19:27,750 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/4cc8b721877044699c0ae5f77e7aa0fc to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/4cc8b721877044699c0ae5f77e7aa0fc 2024-12-02T14:19:27,751 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/90d3b907d20f48d2a64e145cd68d46c0 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/90d3b907d20f48d2a64e145cd68d46c0 2024-12-02T14:19:27,752 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/b75294e5d21f4ee990397da225021bac to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/b75294e5d21f4ee990397da225021bac 2024-12-02T14:19:27,753 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/75c7a56832324f558b9551dcb232cdbb to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/75c7a56832324f558b9551dcb232cdbb 2024-12-02T14:19:27,754 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/3208c6ca895e4655adc5d98b04e2c449 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/3208c6ca895e4655adc5d98b04e2c449 2024-12-02T14:19:27,755 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/e79195bcd8d4486d8c8670db2a69d0ab to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/e79195bcd8d4486d8c8670db2a69d0ab 2024-12-02T14:19:27,757 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/049b71f22c5f4c3b9ce40529e74ba699 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/049b71f22c5f4c3b9ce40529e74ba699 2024-12-02T14:19:27,758 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/8da33d8d63dd4eb7b6d38988896f3ea3 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/8da33d8d63dd4eb7b6d38988896f3ea3 2024-12-02T14:19:27,759 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/0f5be3c37b8d4febbdb6f56f67f61928 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/0f5be3c37b8d4febbdb6f56f67f61928 2024-12-02T14:19:27,760 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c8adcf6419d5497ca1dee1b1e39e5567 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c8adcf6419d5497ca1dee1b1e39e5567 2024-12-02T14:19:27,761 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/36ee31d9f41f4524b05dbd8041163c89 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/36ee31d9f41f4524b05dbd8041163c89 2024-12-02T14:19:27,762 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/fddaa248b2c54423b2fb1d323a307984 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/fddaa248b2c54423b2fb1d323a307984 2024-12-02T14:19:27,763 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/d2acbcd486a54a758ce336acd9a47d3b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/d2acbcd486a54a758ce336acd9a47d3b 2024-12-02T14:19:27,764 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/bc39b0fc210c43a2a311c371f388fdd0 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/bc39b0fc210c43a2a311c371f388fdd0 2024-12-02T14:19:27,765 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/03565537cc5e40f38e4d2075f91c16b5 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/03565537cc5e40f38e4d2075f91c16b5 2024-12-02T14:19:27,778 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/d42455bc8cdb48b7b307d4a87ed05706, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/c80ab0ebd09547dd93b37f0cab988d3d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/525b8a261c1641ae8193282d60781d07, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/a0c1865c20d14894a727716a18596522, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/3750bb6fcb5544039eedf1835570fce7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b905e6ef6fbb4079a6fce303ec7f30e5, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/ea23a104f0f54fb2aa3d9deee20dbe39, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/f5f2801c4fc94e95ab3350249489dd29, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/d7f70141cf1348b48baa5a9ddf4e969d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/da13b012c9fc425f8e7b635ee4760717, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/556a5387681a44c586b1fca4af963fcd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e6a9bc824b89471cb22d4d2b96787062, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/c10b5e535c2a45c6a464dc55aaae97e7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/925e8d4a2c944312a0f4685014523a3e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e1eb1bb7298b4ddd8b98bf72fe428fd6, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e89b002617b94771b049945db70dd978, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/bdcba523237944b0aaee6ed1d0bbec85, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/43f9950d2640433390b7e2731dd43892, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/54484cf7983c4b1eae5bfe40ae1c6e59, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/8e7cc561657b4453832e975c19f3b852, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/410671549eac4efd8b42e7f8ea68867c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/defc3e9abe834e67bc773a7736b445fb, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/9fd40e5f13944201be20318d2dfc8220, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/4177529006c9404b92f8455946b6eaec, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/35746747b5d54cb4b5b6516bf631e09e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/91883128ce564355b06bebd19d378b8d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/88c2f88fbfb84e4d8e0235c4336ffb51, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/88f04dd7b7d54f0bb53e697d1eee322e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b78c1f36d516482cb65becc931f11eb3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/6d3e96cf90d24b76a0421afb87d069eb, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/26d2a5f2a4e64d288971559098d9c008, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/8f6ab76ebb534398aee88e56e58f56f9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/9785bff3af0b433a902be14aebb61ac8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b5bab1e6242c4c78a634edf3b807a038, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/f15dd530d29f4ab0804339d208ef1c03] to archive 2024-12-02T14:19:27,779 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:19:27,781 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/d42455bc8cdb48b7b307d4a87ed05706 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/d42455bc8cdb48b7b307d4a87ed05706 2024-12-02T14:19:27,782 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/c80ab0ebd09547dd93b37f0cab988d3d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/c80ab0ebd09547dd93b37f0cab988d3d 2024-12-02T14:19:27,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/525b8a261c1641ae8193282d60781d07 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/525b8a261c1641ae8193282d60781d07 2024-12-02T14:19:27,785 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/a0c1865c20d14894a727716a18596522 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/a0c1865c20d14894a727716a18596522 2024-12-02T14:19:27,787 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/3750bb6fcb5544039eedf1835570fce7 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/3750bb6fcb5544039eedf1835570fce7 2024-12-02T14:19:27,788 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b905e6ef6fbb4079a6fce303ec7f30e5 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b905e6ef6fbb4079a6fce303ec7f30e5 2024-12-02T14:19:27,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/ea23a104f0f54fb2aa3d9deee20dbe39 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/ea23a104f0f54fb2aa3d9deee20dbe39 2024-12-02T14:19:27,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/f5f2801c4fc94e95ab3350249489dd29 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/f5f2801c4fc94e95ab3350249489dd29 2024-12-02T14:19:27,792 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/d7f70141cf1348b48baa5a9ddf4e969d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/d7f70141cf1348b48baa5a9ddf4e969d 2024-12-02T14:19:27,793 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/da13b012c9fc425f8e7b635ee4760717 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/da13b012c9fc425f8e7b635ee4760717 2024-12-02T14:19:27,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/556a5387681a44c586b1fca4af963fcd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/556a5387681a44c586b1fca4af963fcd 2024-12-02T14:19:27,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e6a9bc824b89471cb22d4d2b96787062 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e6a9bc824b89471cb22d4d2b96787062 2024-12-02T14:19:27,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/c10b5e535c2a45c6a464dc55aaae97e7 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/c10b5e535c2a45c6a464dc55aaae97e7 2024-12-02T14:19:27,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/925e8d4a2c944312a0f4685014523a3e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/925e8d4a2c944312a0f4685014523a3e 2024-12-02T14:19:27,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e1eb1bb7298b4ddd8b98bf72fe428fd6 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e1eb1bb7298b4ddd8b98bf72fe428fd6 2024-12-02T14:19:27,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e89b002617b94771b049945db70dd978 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/e89b002617b94771b049945db70dd978 2024-12-02T14:19:27,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/bdcba523237944b0aaee6ed1d0bbec85 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/bdcba523237944b0aaee6ed1d0bbec85 2024-12-02T14:19:27,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/43f9950d2640433390b7e2731dd43892 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/43f9950d2640433390b7e2731dd43892 2024-12-02T14:19:27,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/54484cf7983c4b1eae5bfe40ae1c6e59 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/54484cf7983c4b1eae5bfe40ae1c6e59 2024-12-02T14:19:27,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/8e7cc561657b4453832e975c19f3b852 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/8e7cc561657b4453832e975c19f3b852 2024-12-02T14:19:27,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/410671549eac4efd8b42e7f8ea68867c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/410671549eac4efd8b42e7f8ea68867c 2024-12-02T14:19:27,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/defc3e9abe834e67bc773a7736b445fb to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/defc3e9abe834e67bc773a7736b445fb 2024-12-02T14:19:27,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/9fd40e5f13944201be20318d2dfc8220 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/9fd40e5f13944201be20318d2dfc8220 2024-12-02T14:19:27,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/4177529006c9404b92f8455946b6eaec to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/4177529006c9404b92f8455946b6eaec 2024-12-02T14:19:27,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/35746747b5d54cb4b5b6516bf631e09e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/35746747b5d54cb4b5b6516bf631e09e 2024-12-02T14:19:27,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/91883128ce564355b06bebd19d378b8d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/91883128ce564355b06bebd19d378b8d 2024-12-02T14:19:27,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/88c2f88fbfb84e4d8e0235c4336ffb51 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/88c2f88fbfb84e4d8e0235c4336ffb51 2024-12-02T14:19:27,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/88f04dd7b7d54f0bb53e697d1eee322e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/88f04dd7b7d54f0bb53e697d1eee322e 2024-12-02T14:19:27,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b78c1f36d516482cb65becc931f11eb3 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b78c1f36d516482cb65becc931f11eb3 2024-12-02T14:19:27,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/6d3e96cf90d24b76a0421afb87d069eb to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/6d3e96cf90d24b76a0421afb87d069eb 2024-12-02T14:19:27,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/26d2a5f2a4e64d288971559098d9c008 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/26d2a5f2a4e64d288971559098d9c008 2024-12-02T14:19:27,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/8f6ab76ebb534398aee88e56e58f56f9 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/8f6ab76ebb534398aee88e56e58f56f9 2024-12-02T14:19:27,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/9785bff3af0b433a902be14aebb61ac8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/9785bff3af0b433a902be14aebb61ac8 2024-12-02T14:19:27,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b5bab1e6242c4c78a634edf3b807a038 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b5bab1e6242c4c78a634edf3b807a038 2024-12-02T14:19:27,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/f15dd530d29f4ab0804339d208ef1c03 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/f15dd530d29f4ab0804339d208ef1c03 2024-12-02T14:19:27,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/951a59975b504d3c88e22c28c20e76c4, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/11b36ffe8f464d8b8a9ef87d1c751baa, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/ed5a32a7bdeb473ca5bc6f32202ed3df, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/253b2a899518423bb14f66f9ccc18c6b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/da2d5efa98394bc3969f49fc89acfd6f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/75182915b71245179f00ed51f8616878, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/75262c3f425040f4b26f46c11aea7cc0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/3857f165dfd3425cb0327b590994dc1f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/05ff48dc5b134771b0bb7c19f1ba7bba, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/d6c8fbd16acc45889af586ec1cef26c0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2b92a56044bb4644beed7d3677603759, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/1a9fd5eb1fc346b5a930b8448706efa2, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b390a727326c40aa9171896820d53480, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2cd0dd2b7a2b4d83bb816cc64ec1f115, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2e7f57b562a643c889d3f4b15dd3e67a, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/bb3b88843a9146aa8afb39fe89b66384, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/a9de6b9ccfb1433cb5e01a5d8e9924eb, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/21b6fceafea74ede99138eab4882a180, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/463a72b549aa4d31b9924c78b3a6b85d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/a1210443e48a430ba2ca291fa0a5b10b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2205925f3809492fb27a2b790618704c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/193f9db2bff340eea13d6d2878ddec0c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/14e4a1ecd0764dc7a0ae855fd29aed0b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/90c6f9e147d249b68128bf1f6baa421f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/62241b72102a46ea845d2686368ff7d8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/faa08da2cc15460d8da7f80249724924, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/f622ae2c72cf4a929ff236140d6c96ef, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/01d35e26d5354baba7883fe03149ac0a, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/c882676449aa499c9253b166481be1c1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/7469a00a984042d7b71994ae2323a8ec, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/9a1145dcb1fd41eea3d85b5262a6cf86, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b8d02a77f27b4109bb93537b577566a2, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b5155ded76cc43c1bacd721bfcc96acf, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2ab3bf97718546ea9dc7db7db4123ff7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b7fdc9058a5f4d87b21ef9807c32c0b8] to archive 2024-12-02T14:19:27,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:19:27,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/951a59975b504d3c88e22c28c20e76c4 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/951a59975b504d3c88e22c28c20e76c4 2024-12-02T14:19:27,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/11b36ffe8f464d8b8a9ef87d1c751baa to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/11b36ffe8f464d8b8a9ef87d1c751baa 2024-12-02T14:19:27,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/ed5a32a7bdeb473ca5bc6f32202ed3df to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/ed5a32a7bdeb473ca5bc6f32202ed3df 2024-12-02T14:19:27,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/253b2a899518423bb14f66f9ccc18c6b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/253b2a899518423bb14f66f9ccc18c6b 2024-12-02T14:19:27,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/da2d5efa98394bc3969f49fc89acfd6f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/da2d5efa98394bc3969f49fc89acfd6f 2024-12-02T14:19:27,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/75182915b71245179f00ed51f8616878 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/75182915b71245179f00ed51f8616878 2024-12-02T14:19:27,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/75262c3f425040f4b26f46c11aea7cc0 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/75262c3f425040f4b26f46c11aea7cc0 2024-12-02T14:19:27,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/3857f165dfd3425cb0327b590994dc1f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/3857f165dfd3425cb0327b590994dc1f 2024-12-02T14:19:27,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/05ff48dc5b134771b0bb7c19f1ba7bba to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/05ff48dc5b134771b0bb7c19f1ba7bba 2024-12-02T14:19:27,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/d6c8fbd16acc45889af586ec1cef26c0 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/d6c8fbd16acc45889af586ec1cef26c0 2024-12-02T14:19:27,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2b92a56044bb4644beed7d3677603759 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2b92a56044bb4644beed7d3677603759 2024-12-02T14:19:27,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/1a9fd5eb1fc346b5a930b8448706efa2 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/1a9fd5eb1fc346b5a930b8448706efa2 2024-12-02T14:19:27,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b390a727326c40aa9171896820d53480 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b390a727326c40aa9171896820d53480 2024-12-02T14:19:27,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2cd0dd2b7a2b4d83bb816cc64ec1f115 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2cd0dd2b7a2b4d83bb816cc64ec1f115 2024-12-02T14:19:27,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2e7f57b562a643c889d3f4b15dd3e67a to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2e7f57b562a643c889d3f4b15dd3e67a 2024-12-02T14:19:27,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/bb3b88843a9146aa8afb39fe89b66384 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/bb3b88843a9146aa8afb39fe89b66384 2024-12-02T14:19:27,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/a9de6b9ccfb1433cb5e01a5d8e9924eb to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/a9de6b9ccfb1433cb5e01a5d8e9924eb 2024-12-02T14:19:27,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/21b6fceafea74ede99138eab4882a180 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/21b6fceafea74ede99138eab4882a180 2024-12-02T14:19:27,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/463a72b549aa4d31b9924c78b3a6b85d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/463a72b549aa4d31b9924c78b3a6b85d 2024-12-02T14:19:27,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/a1210443e48a430ba2ca291fa0a5b10b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/a1210443e48a430ba2ca291fa0a5b10b 2024-12-02T14:19:27,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2205925f3809492fb27a2b790618704c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2205925f3809492fb27a2b790618704c 2024-12-02T14:19:27,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/193f9db2bff340eea13d6d2878ddec0c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/193f9db2bff340eea13d6d2878ddec0c 2024-12-02T14:19:27,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/14e4a1ecd0764dc7a0ae855fd29aed0b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/14e4a1ecd0764dc7a0ae855fd29aed0b 2024-12-02T14:19:27,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/90c6f9e147d249b68128bf1f6baa421f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/90c6f9e147d249b68128bf1f6baa421f 2024-12-02T14:19:27,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/62241b72102a46ea845d2686368ff7d8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/62241b72102a46ea845d2686368ff7d8 2024-12-02T14:19:27,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/faa08da2cc15460d8da7f80249724924 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/faa08da2cc15460d8da7f80249724924 2024-12-02T14:19:27,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/f622ae2c72cf4a929ff236140d6c96ef to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/f622ae2c72cf4a929ff236140d6c96ef 2024-12-02T14:19:27,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/01d35e26d5354baba7883fe03149ac0a to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/01d35e26d5354baba7883fe03149ac0a 2024-12-02T14:19:27,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/c882676449aa499c9253b166481be1c1 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/c882676449aa499c9253b166481be1c1 2024-12-02T14:19:27,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/7469a00a984042d7b71994ae2323a8ec to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/7469a00a984042d7b71994ae2323a8ec 2024-12-02T14:19:27,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/9a1145dcb1fd41eea3d85b5262a6cf86 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/9a1145dcb1fd41eea3d85b5262a6cf86 2024-12-02T14:19:27,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b8d02a77f27b4109bb93537b577566a2 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b8d02a77f27b4109bb93537b577566a2 2024-12-02T14:19:27,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b5155ded76cc43c1bacd721bfcc96acf to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b5155ded76cc43c1bacd721bfcc96acf 2024-12-02T14:19:27,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2ab3bf97718546ea9dc7db7db4123ff7 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/2ab3bf97718546ea9dc7db7db4123ff7 2024-12-02T14:19:27,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b7fdc9058a5f4d87b21ef9807c32c0b8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/b7fdc9058a5f4d87b21ef9807c32c0b8 2024-12-02T14:19:27,895 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/recovered.edits/560.seqid, newMaxSeqId=560, maxSeqId=1 2024-12-02T14:19:27,896 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. 2024-12-02T14:19:27,896 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 4eaa960f4ddde5e2082f5ac5eba8cec8: Waiting for close lock at 1733149166374Running coprocessor pre-close hooks at 1733149166374Disabling compacts and flushes for region at 1733149166374Disabling writes for close at 1733149166453 (+79 ms)Obtaining lock to block concurrent updates at 1733149166454 (+1 ms)Preparing flush snapshotting stores in 4eaa960f4ddde5e2082f5ac5eba8cec8 at 1733149166454Finished memstore snapshotting TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733149166454Flushing stores of TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8. at 1733149166455 (+1 ms)Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8/A: creating writer at 1733149166455Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8/A: appending metadata at 1733149166458 (+3 ms)Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8/A: closing flushed file at 1733149166458Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8/B: creating writer at 1733149166871 (+413 ms)Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8/B: appending metadata at 1733149166876 (+5 ms)Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8/B: closing flushed file at 1733149166876Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8/C: creating writer at 1733149167285 (+409 ms)Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8/C: appending metadata at 1733149167288 (+3 ms)Flushing 4eaa960f4ddde5e2082f5ac5eba8cec8/C: closing flushed file at 1733149167288Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4218ae3d: reopening flushed file at 1733149167698 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51f5a9f: reopening flushed file at 1733149167704 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@319ef690: reopening flushed file at 1733149167709 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4eaa960f4ddde5e2082f5ac5eba8cec8 in 1264ms, sequenceid=557, compaction requested=false at 1733149167718 (+9 ms)Writing region close event to WAL at 1733149167891 (+173 ms)Running coprocessor post-close hooks at 1733149167896 (+5 ms)Closed at 1733149167896 2024-12-02T14:19:27,898 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:27,899 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=4eaa960f4ddde5e2082f5ac5eba8cec8, regionState=CLOSED 2024-12-02T14:19:27,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:19:27,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-02T14:19:27,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 4eaa960f4ddde5e2082f5ac5eba8cec8, server=a3a61c9ba14f,46793,1733149138543 in 1.6890 sec 2024-12-02T14:19:27,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-02T14:19:27,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4eaa960f4ddde5e2082f5ac5eba8cec8, UNASSIGN in 1.6960 sec 2024-12-02T14:19:27,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-02T14:19:27,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7020 sec 2024-12-02T14:19:27,908 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149167908"}]},"ts":"1733149167908"} 2024-12-02T14:19:27,909 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T14:19:27,909 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T14:19:27,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7170 sec 2024-12-02T14:19:28,273 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-02T14:19:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T14:19:28,349 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:28,354 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T14:19:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:19:28,363 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:19:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-02T14:19:28,365 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:19:28,370 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:28,373 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/recovered.edits] 2024-12-02T14:19:28,377 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/3bdfa7fb8b4f42b89167a48b31e032e0 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/3bdfa7fb8b4f42b89167a48b31e032e0 2024-12-02T14:19:28,379 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c3d608fb582b48199ee84d57797edfce to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/A/c3d608fb582b48199ee84d57797edfce 2024-12-02T14:19:28,382 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/88061035ccfa435a9249780bd4fafd7e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/88061035ccfa435a9249780bd4fafd7e 2024-12-02T14:19:28,384 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b6b00622595f4f86aafb5713c2074418 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/B/b6b00622595f4f86aafb5713c2074418 2024-12-02T14:19:28,387 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/6af8276460a04405ae4d316ebc49e5d5 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/6af8276460a04405ae4d316ebc49e5d5 2024-12-02T14:19:28,389 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/a420c9aaef1843a9b26d75b68fdc1a05 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/C/a420c9aaef1843a9b26d75b68fdc1a05 2024-12-02T14:19:28,392 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/recovered.edits/560.seqid to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8/recovered.edits/560.seqid 2024-12-02T14:19:28,392 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/4eaa960f4ddde5e2082f5ac5eba8cec8 2024-12-02T14:19:28,392 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T14:19:28,397 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:19:28,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-02T14:19:28,406 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T14:19:28,441 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T14:19:28,442 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:19:28,443 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T14:19:28,443 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733149168443"}]},"ts":"9223372036854775807"} 2024-12-02T14:19:28,447 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T14:19:28,447 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4eaa960f4ddde5e2082f5ac5eba8cec8, NAME => 'TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T14:19:28,447 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T14:19:28,447 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733149168447"}]},"ts":"9223372036854775807"} 2024-12-02T14:19:28,450 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T14:19:28,451 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:19:28,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 95 msec 2024-12-02T14:19:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-02T14:19:28,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T14:19:28,469 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:28,484 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 210) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-938368545_22 at /127.0.0.1:56006 [Waiting for operation #246] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_180533634_22 at /127.0.0.1:36810 [Waiting for operation #227] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-938368545_22 at /127.0.0.1:36800 [Waiting for operation #228] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;a3a61c9ba14f:46793-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=449 (was 321) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6205 (was 5669) - AvailableMemoryMB LEAK? - 2024-12-02T14:19:28,495 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=449, ProcessCount=11, AvailableMemoryMB=6205 2024-12-02T14:19:28,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T14:19:28,497 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T14:19:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T14:19:28,499 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T14:19:28,499 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:28,499 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-12-02T14:19:28,500 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T14:19:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-02T14:19:28,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741948_1124 (size=1061) 2024-12-02T14:19:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-02T14:19:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-02T14:19:28,907 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919 2024-12-02T14:19:28,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741949_1125 (size=53) 2024-12-02T14:19:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-02T14:19:29,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:19:29,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 367203f21af3d1a942fdfc82bb3c32fd, disabling compactions & flushes 2024-12-02T14:19:29,313 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:29,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:29,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. after waiting 0 ms 2024-12-02T14:19:29,313 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:29,313 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:29,314 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 367203f21af3d1a942fdfc82bb3c32fd: Waiting for close lock at 1733149169313Disabling compacts and flushes for region at 1733149169313Disabling writes for close at 1733149169313Writing region close event to WAL at 1733149169313Closed at 1733149169313 2024-12-02T14:19:29,315 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T14:19:29,315 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733149169315"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733149169315"}]},"ts":"1733149169315"} 2024-12-02T14:19:29,316 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T14:19:29,317 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T14:19:29,317 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149169317"}]},"ts":"1733149169317"} 2024-12-02T14:19:29,319 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T14:19:29,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=367203f21af3d1a942fdfc82bb3c32fd, ASSIGN}] 2024-12-02T14:19:29,320 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=367203f21af3d1a942fdfc82bb3c32fd, ASSIGN 2024-12-02T14:19:29,321 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=367203f21af3d1a942fdfc82bb3c32fd, ASSIGN; state=OFFLINE, location=a3a61c9ba14f,46793,1733149138543; forceNewPlan=false, retain=false 2024-12-02T14:19:29,471 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=367203f21af3d1a942fdfc82bb3c32fd, regionState=OPENING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:29,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=367203f21af3d1a942fdfc82bb3c32fd, ASSIGN because future has completed 2024-12-02T14:19:29,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:19:29,633 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:29,634 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} 2024-12-02T14:19:29,634 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:29,634 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:19:29,635 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:29,635 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:29,637 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-02T14:19:29,639 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:19:29,640 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 367203f21af3d1a942fdfc82bb3c32fd columnFamilyName A 2024-12-02T14:19:29,641 DEBUG [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:29,641 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.HStore(327): Store=367203f21af3d1a942fdfc82bb3c32fd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:19:29,642 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:29,643 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:19:29,644 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 367203f21af3d1a942fdfc82bb3c32fd columnFamilyName B 2024-12-02T14:19:29,644 DEBUG [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:29,644 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.HStore(327): Store=367203f21af3d1a942fdfc82bb3c32fd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:19:29,644 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:29,645 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:19:29,646 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 367203f21af3d1a942fdfc82bb3c32fd columnFamilyName C 2024-12-02T14:19:29,646 DEBUG [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:29,646 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.HStore(327): Store=367203f21af3d1a942fdfc82bb3c32fd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:19:29,646 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:29,646 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:29,647 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:29,647 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:29,648 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:29,648 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:29,649 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T14:19:29,650 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:29,652 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T14:19:29,652 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 367203f21af3d1a942fdfc82bb3c32fd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68314943, jitterRate=0.017971977591514587}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T14:19:29,653 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:29,653 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 367203f21af3d1a942fdfc82bb3c32fd: Running coprocessor pre-open hook at 1733149169635Writing region info on filesystem at 1733149169635Initializing all the Stores at 1733149169636 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149169636Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149169637 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149169637Cleaning up temporary data from old regions at 1733149169649 (+12 ms)Running coprocessor post-open hooks at 1733149169653 (+4 ms)Region opened successfully at 1733149169653 2024-12-02T14:19:29,654 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., pid=34, masterSystemTime=1733149169629 2024-12-02T14:19:29,656 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:29,656 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:29,656 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=367203f21af3d1a942fdfc82bb3c32fd, regionState=OPEN, openSeqNum=2, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:29,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:19:29,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-02T14:19:29,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 in 183 msec 2024-12-02T14:19:29,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-02T14:19:29,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=367203f21af3d1a942fdfc82bb3c32fd, ASSIGN in 342 msec 2024-12-02T14:19:29,663 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T14:19:29,663 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149169663"}]},"ts":"1733149169663"} 2024-12-02T14:19:29,665 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T14:19:29,666 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T14:19:29,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-12-02T14:19:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-02T14:19:30,650 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:30,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594c9324, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:30,652 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:30,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:30,655 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d3d65be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:30,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:30,658 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@659acc76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:30,659 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:30,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:30,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:30,663 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:30,664 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T14:19:30,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is a3a61c9ba14f,35877,1733149137941 2024-12-02T14:19:30,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@51b94d83 2024-12-02T14:19:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T14:19:30,667 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T14:19:30,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T14:19:30,673 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T14:19:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T14:19:30,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741950_1126 (size=1097) 2024-12-02T14:19:31,092 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T14:19:31,093 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T14:19:31,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T14:19:31,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=367203f21af3d1a942fdfc82bb3c32fd, REOPEN/MOVE}] 2024-12-02T14:19:31,107 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=367203f21af3d1a942fdfc82bb3c32fd, REOPEN/MOVE 2024-12-02T14:19:31,107 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=367203f21af3d1a942fdfc82bb3c32fd, regionState=CLOSING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:31,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=367203f21af3d1a942fdfc82bb3c32fd, REOPEN/MOVE because future has completed 2024-12-02T14:19:31,109 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T14:19:31,109 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:19:31,262 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:31,263 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T14:19:31,263 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 367203f21af3d1a942fdfc82bb3c32fd, disabling compactions & flushes 2024-12-02T14:19:31,263 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:31,263 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:31,263 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. after waiting 0 ms 2024-12-02T14:19:31,263 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:31,270 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T14:19:31,271 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:31,272 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 367203f21af3d1a942fdfc82bb3c32fd: Waiting for close lock at 1733149171263Running coprocessor pre-close hooks at 1733149171263Disabling compacts and flushes for region at 1733149171263Disabling writes for close at 1733149171263Writing region close event to WAL at 1733149171264 (+1 ms)Running coprocessor post-close hooks at 1733149171271 (+7 ms)Closed at 1733149171271 2024-12-02T14:19:31,272 WARN [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 367203f21af3d1a942fdfc82bb3c32fd to self. 2024-12-02T14:19:31,274 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:31,275 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=367203f21af3d1a942fdfc82bb3c32fd, regionState=CLOSED 2024-12-02T14:19:31,276 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:19:31,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-02T14:19:31,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 in 169 msec 2024-12-02T14:19:31,280 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=367203f21af3d1a942fdfc82bb3c32fd, REOPEN/MOVE; state=CLOSED, location=a3a61c9ba14f,46793,1733149138543; forceNewPlan=false, retain=true 2024-12-02T14:19:31,432 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=367203f21af3d1a942fdfc82bb3c32fd, regionState=OPENING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:31,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=367203f21af3d1a942fdfc82bb3c32fd, REOPEN/MOVE because future has completed 2024-12-02T14:19:31,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:19:31,594 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:31,594 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} 2024-12-02T14:19:31,594 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:31,594 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:19:31,594 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:31,594 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:31,597 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:31,597 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:19:31,602 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 367203f21af3d1a942fdfc82bb3c32fd columnFamilyName A 2024-12-02T14:19:31,604 DEBUG [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:31,604 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.HStore(327): Store=367203f21af3d1a942fdfc82bb3c32fd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:19:31,605 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:31,605 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:19:31,605 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 367203f21af3d1a942fdfc82bb3c32fd columnFamilyName B 2024-12-02T14:19:31,605 DEBUG [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:31,606 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.HStore(327): Store=367203f21af3d1a942fdfc82bb3c32fd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:19:31,606 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:31,607 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:19:31,607 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 367203f21af3d1a942fdfc82bb3c32fd columnFamilyName C 2024-12-02T14:19:31,607 DEBUG [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:31,607 INFO [StoreOpener-367203f21af3d1a942fdfc82bb3c32fd-1 {}] regionserver.HStore(327): Store=367203f21af3d1a942fdfc82bb3c32fd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:19:31,607 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:31,607 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:31,608 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:31,609 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:31,610 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:31,610 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:31,610 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T14:19:31,611 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:31,612 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 367203f21af3d1a942fdfc82bb3c32fd; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73873446, jitterRate=0.10080012679100037}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T14:19:31,612 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:31,613 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 367203f21af3d1a942fdfc82bb3c32fd: Running coprocessor pre-open hook at 1733149171594Writing region info on filesystem at 1733149171594Initializing all the Stores at 1733149171595 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149171595Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149171596 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149171596Cleaning up temporary data from old regions at 1733149171610 (+14 ms)Running coprocessor post-open hooks at 1733149171612 (+2 ms)Region opened successfully at 1733149171613 (+1 ms) 2024-12-02T14:19:31,614 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., pid=39, masterSystemTime=1733149171591 2024-12-02T14:19:31,615 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:31,615 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:31,616 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=367203f21af3d1a942fdfc82bb3c32fd, regionState=OPEN, openSeqNum=5, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:31,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:19:31,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-12-02T14:19:31,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 in 181 msec 2024-12-02T14:19:31,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-02T14:19:31,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=367203f21af3d1a942fdfc82bb3c32fd, REOPEN/MOVE in 514 msec 2024-12-02T14:19:31,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-02T14:19:31,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 523 msec 2024-12-02T14:19:31,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 950 msec 2024-12-02T14:19:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-02T14:19:31,629 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:31,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fcb20d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:31,629 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:31,630 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:31,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:31,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6039fe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:31,632 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:31,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eb2646, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e853879, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:31,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:31,635 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ccc0fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:31,636 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:31,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6de256f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a00ea56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,638 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:31,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:31,639 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31539f9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:31,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:31,640 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:31,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b09a0bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636d090d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:31,642 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:31,643 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69789e13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:31,644 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:31,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4658be7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e0133ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:31,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:31,646 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:31,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ebb2eca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:31,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:31,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:31,648 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:31,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ad78b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc5db83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,649 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:31,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:31,650 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54582682, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:31,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:31,651 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48d6cc2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4870d163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:31,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:31,654 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2014590d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:31,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:31,655 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:31,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a36ecf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53e9634b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:31,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:31,658 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25bbb736, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:31,659 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:31,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c4a99b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f7e914, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,660 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:19:31,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:19:31,661 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:19:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:19:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:19:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c789e7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:19:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:19:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:31,663 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:19:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1713fe00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:19:31,664 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:31,664 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:31,664 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:31,664 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:31,665 DEBUG [Thread-643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:31,665 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:31,665 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:31,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-02T14:19:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:31,666 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:31,667 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:31,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:31,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T14:19:31,668 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:19:31,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:31,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:31,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:31,670 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:31,670 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:31,670 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:31,670 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:31,670 DEBUG [Thread-643 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:19:31,671 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:31,671 DEBUG [Thread-643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:19:31,671 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:31,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:19:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:19:31,672 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:31,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:31,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:19:31,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:19:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:19:31,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:19:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:19:31,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:19:31,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:19:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:19:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:31,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T14:19:31,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:31,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:31,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:31,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:31,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:31,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:31,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d6947e12cca846d389154949bfd9adb9_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149171688/Put/seqid=0 2024-12-02T14:19:31,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741951_1127 (size=17034) 2024-12-02T14:19:31,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733149171722, max timestamp=1733149171750Num uniques 859; 2024-12-02T14:19:31,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733149171722, max timestamp=1733149171750Num uniques 859; 2024-12-02T14:19:31,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T14:19:31,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T14:19:31,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733149171722, max timestamp=1733149171750Num uniques 859; 2024-12-02T14:19:31,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T14:19:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T14:19:31,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:31,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149231789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:31,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:31,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48354 deadline: 1733149231793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:31,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:31,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149231793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:31,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:31,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149231795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:31,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:31,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149231795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:31,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T14:19:31,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:31,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:31,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:31,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:31,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:31,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:31,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:31,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149231899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:31,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:31,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48354 deadline: 1733149231908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:31,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149231908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:31,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149231919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:31,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149231919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:31,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T14:19:31,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:31,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:31,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:31,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:31,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:31,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T14:19:32,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149232112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48354 deadline: 1733149232120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149232129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149232129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149232129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T14:19:32,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:32,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:32,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:32,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:32,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:32,142 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,149 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d6947e12cca846d389154949bfd9adb9_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d6947e12cca846d389154949bfd9adb9_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:32,152 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/18c40e20d4f6433e85ed1ea64000a427, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:32,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/18c40e20d4f6433e85ed1ea64000a427 is 175, key is test_row_0/A:col10/1733149171688/Put/seqid=0 2024-12-02T14:19:32,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741952_1128 (size=48139) 2024-12-02T14:19:32,170 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/18c40e20d4f6433e85ed1ea64000a427 2024-12-02T14:19:32,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/9db191b33da54b3da7e195a5f4e3110e is 50, key is test_row_0/B:col10/1733149171688/Put/seqid=0 2024-12-02T14:19:32,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741953_1129 (size=12001) 2024-12-02T14:19:32,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/9db191b33da54b3da7e195a5f4e3110e 2024-12-02T14:19:32,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/4e7670f601284f028a0b2aec428a04d8 is 50, key is test_row_0/C:col10/1733149171688/Put/seqid=0 2024-12-02T14:19:32,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741954_1130 (size=12001) 2024-12-02T14:19:32,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/4e7670f601284f028a0b2aec428a04d8 2024-12-02T14:19:32,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/18c40e20d4f6433e85ed1ea64000a427 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/18c40e20d4f6433e85ed1ea64000a427 2024-12-02T14:19:32,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T14:19:32,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:32,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:32,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:32,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:32,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:32,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/18c40e20d4f6433e85ed1ea64000a427, entries=250, sequenceid=14, filesize=47.0 K 2024-12-02T14:19:32,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/9db191b33da54b3da7e195a5f4e3110e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/9db191b33da54b3da7e195a5f4e3110e 2024-12-02T14:19:32,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T14:19:32,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/9db191b33da54b3da7e195a5f4e3110e, entries=150, sequenceid=14, filesize=11.7 K 2024-12-02T14:19:32,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/4e7670f601284f028a0b2aec428a04d8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4e7670f601284f028a0b2aec428a04d8 2024-12-02T14:19:32,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4e7670f601284f028a0b2aec428a04d8, entries=150, sequenceid=14, filesize=11.7 K 2024-12-02T14:19:32,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 367203f21af3d1a942fdfc82bb3c32fd in 634ms, sequenceid=14, compaction requested=false 2024-12-02T14:19:32,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:32,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-12-02T14:19:32,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:32,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:32,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:32,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:32,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:32,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:32,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T14:19:32,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:32,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:32,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:32,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:32,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:32,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202deff486488594c16abc77b890e7799f7_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149171792/Put/seqid=0 2024-12-02T14:19:32,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149232453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149232455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149232458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48354 deadline: 1733149232459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149232459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741955_1131 (size=14594) 2024-12-02T14:19:32,473 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,480 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202deff486488594c16abc77b890e7799f7_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202deff486488594c16abc77b890e7799f7_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:32,482 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/b0cbd176fa874ec0b604ce6fab358d43, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:32,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/b0cbd176fa874ec0b604ce6fab358d43 is 175, key is test_row_0/A:col10/1733149171792/Put/seqid=0 2024-12-02T14:19:32,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741956_1132 (size=39549) 2024-12-02T14:19:32,513 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/b0cbd176fa874ec0b604ce6fab358d43 2024-12-02T14:19:32,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/5794c44ad3ce42609955111f2c71eb29 is 50, key is test_row_0/B:col10/1733149171792/Put/seqid=0 2024-12-02T14:19:32,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741957_1133 (size=12001) 2024-12-02T14:19:32,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/5794c44ad3ce42609955111f2c71eb29 2024-12-02T14:19:32,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/9f8108063ca943d596af44827d3cad63 is 50, key is test_row_0/C:col10/1733149171792/Put/seqid=0 2024-12-02T14:19:32,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149232569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149232569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149232569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48354 deadline: 1733149232570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741958_1134 (size=12001) 2024-12-02T14:19:32,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/9f8108063ca943d596af44827d3cad63 2024-12-02T14:19:32,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149232580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/b0cbd176fa874ec0b604ce6fab358d43 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/b0cbd176fa874ec0b604ce6fab358d43 2024-12-02T14:19:32,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T14:19:32,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:32,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:32,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:32,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:32,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:32,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:32,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/b0cbd176fa874ec0b604ce6fab358d43, entries=200, sequenceid=51, filesize=38.6 K 2024-12-02T14:19:32,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/5794c44ad3ce42609955111f2c71eb29 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5794c44ad3ce42609955111f2c71eb29 2024-12-02T14:19:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5794c44ad3ce42609955111f2c71eb29, entries=150, sequenceid=51, filesize=11.7 K 2024-12-02T14:19:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/9f8108063ca943d596af44827d3cad63 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/9f8108063ca943d596af44827d3cad63 2024-12-02T14:19:32,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/9f8108063ca943d596af44827d3cad63, entries=150, sequenceid=51, filesize=11.7 K 2024-12-02T14:19:32,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=46.96 KB/48090 for 367203f21af3d1a942fdfc82bb3c32fd in 197ms, sequenceid=51, compaction requested=false 2024-12-02T14:19:32,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T14:19:32,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:32,753 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T14:19:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:32,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:32,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:32,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:32,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120231b6768510464063ab9eb48d3d240be5_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149172455/Put/seqid=0 2024-12-02T14:19:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741959_1135 (size=9714) 2024-12-02T14:19:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,775 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120231b6768510464063ab9eb48d3d240be5_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120231b6768510464063ab9eb48d3d240be5_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/d8aa232c5064464ba964eb849a3403ed, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/d8aa232c5064464ba964eb849a3403ed is 175, key is test_row_0/A:col10/1733149172455/Put/seqid=0 2024-12-02T14:19:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:32,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:32,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T14:19:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741960_1136 (size=22361) 2024-12-02T14:19:32,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,820 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/d8aa232c5064464ba964eb849a3403ed 2024-12-02T14:19:32,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:32,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149232831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733149172815, max timestamp=1733149172835Num uniques 859; 2024-12-02T14:19:32,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T14:19:32,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733149172815, max timestamp=1733149172835Num uniques 859; 2024-12-02T14:19:32,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T14:19:32,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/3d7b0fb8086346beb5726ffeac283d70 is 50, key is test_row_0/B:col10/1733149172455/Put/seqid=0 2024-12-02T14:19:32,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733149172815, max timestamp=1733149172835Num uniques 859; 2024-12-02T14:19:32,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T14:19:32,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741961_1137 (size=9657) 2024-12-02T14:19:32,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149232870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149232871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149232871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48354 deadline: 1733149232872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149232949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149232989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48354 deadline: 1733149232989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149232989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149232989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:32,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:33,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:33,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149233159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:33,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:33,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149233198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:33,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:33,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48354 deadline: 1733149233198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:33,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149233199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:33,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149233208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:33,249 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/3d7b0fb8086346beb5726ffeac283d70 2024-12-02T14:19:33,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/76b7e43c322048c889bf39f4435b175e is 50, key is test_row_0/C:col10/1733149172455/Put/seqid=0 2024-12-02T14:19:33,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741962_1138 (size=9657) 2024-12-02T14:19:33,272 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/76b7e43c322048c889bf39f4435b175e 2024-12-02T14:19:33,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/d8aa232c5064464ba964eb849a3403ed as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/d8aa232c5064464ba964eb849a3403ed 2024-12-02T14:19:33,284 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/d8aa232c5064464ba964eb849a3403ed, entries=100, sequenceid=61, filesize=21.8 K 2024-12-02T14:19:33,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/3d7b0fb8086346beb5726ffeac283d70 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3d7b0fb8086346beb5726ffeac283d70 2024-12-02T14:19:33,291 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3d7b0fb8086346beb5726ffeac283d70, entries=100, sequenceid=61, filesize=9.4 K 2024-12-02T14:19:33,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/76b7e43c322048c889bf39f4435b175e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/76b7e43c322048c889bf39f4435b175e 2024-12-02T14:19:33,300 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/76b7e43c322048c889bf39f4435b175e, entries=100, sequenceid=61, filesize=9.4 K 2024-12-02T14:19:33,301 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 367203f21af3d1a942fdfc82bb3c32fd in 548ms, sequenceid=61, compaction requested=true 2024-12-02T14:19:33,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:33,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:33,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-02T14:19:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-02T14:19:33,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-02T14:19:33,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6310 sec 2024-12-02T14:19:33,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.6460 sec 2024-12-02T14:19:33,428 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T14:19:33,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:33,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-02T14:19:33,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:33,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:33,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:33,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:33,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:33,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:33,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202855f7cbdac9c4b30a39144351640bf68_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149173472/Put/seqid=0 2024-12-02T14:19:33,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48354 deadline: 1733149233509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:33,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149233509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:33,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:33,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149233509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:33,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149233512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:33,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:33,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149233518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:33,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:33,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741963_1139 (size=14594) 2024-12-02T14:19:33,528 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:33,534 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202855f7cbdac9c4b30a39144351640bf68_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202855f7cbdac9c4b30a39144351640bf68_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:33,536 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/1900f7f47e704698bfca5dafd22c22f5, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:33,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/1900f7f47e704698bfca5dafd22c22f5 is 175, key is test_row_0/A:col10/1733149173472/Put/seqid=0 2024-12-02T14:19:33,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741964_1140 (size=39549) 2024-12-02T14:19:33,554 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/1900f7f47e704698bfca5dafd22c22f5 2024-12-02T14:19:33,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/cfb013639aa942c2984365259859316b is 50, key is test_row_0/B:col10/1733149173472/Put/seqid=0 2024-12-02T14:19:33,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741965_1141 (size=12001) 2024-12-02T14:19:33,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/cfb013639aa942c2984365259859316b 2024-12-02T14:19:33,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/b961a273a0c444dcbb5927f37a650178 is 50, key is test_row_0/C:col10/1733149173472/Put/seqid=0 2024-12-02T14:19:33,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741966_1142 (size=12001) 2024-12-02T14:19:33,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/b961a273a0c444dcbb5927f37a650178 2024-12-02T14:19:33,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/1900f7f47e704698bfca5dafd22c22f5 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/1900f7f47e704698bfca5dafd22c22f5 2024-12-02T14:19:33,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/1900f7f47e704698bfca5dafd22c22f5, entries=200, sequenceid=97, filesize=38.6 K 2024-12-02T14:19:33,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149233630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:33,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:33,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:33,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/cfb013639aa942c2984365259859316b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/cfb013639aa942c2984365259859316b 2024-12-02T14:19:33,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/cfb013639aa942c2984365259859316b, entries=150, sequenceid=97, filesize=11.7 K 2024-12-02T14:19:33,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/b961a273a0c444dcbb5927f37a650178 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/b961a273a0c444dcbb5927f37a650178 2024-12-02T14:19:33,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/b961a273a0c444dcbb5927f37a650178, entries=150, sequenceid=97, filesize=11.7 K 2024-12-02T14:19:33,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 367203f21af3d1a942fdfc82bb3c32fd in 174ms, sequenceid=97, compaction requested=true 2024-12-02T14:19:33,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:33,649 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:33,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:33,650 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:33,650 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/A is initiating minor compaction (all files) 2024-12-02T14:19:33,651 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/A in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:33,651 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/18c40e20d4f6433e85ed1ea64000a427, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/b0cbd176fa874ec0b604ce6fab358d43, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/d8aa232c5064464ba964eb849a3403ed, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/1900f7f47e704698bfca5dafd22c22f5] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=146.1 K 2024-12-02T14:19:33,651 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:33,651 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/18c40e20d4f6433e85ed1ea64000a427, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/b0cbd176fa874ec0b604ce6fab358d43, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/d8aa232c5064464ba964eb849a3403ed, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/1900f7f47e704698bfca5dafd22c22f5] 2024-12-02T14:19:33,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:33,652 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18c40e20d4f6433e85ed1ea64000a427, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733149171677 2024-12-02T14:19:33,653 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0cbd176fa874ec0b604ce6fab358d43, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733149171788 2024-12-02T14:19:33,653 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8aa232c5064464ba964eb849a3403ed, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733149172455 2024-12-02T14:19:33,654 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:33,654 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1900f7f47e704698bfca5dafd22c22f5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733149172856 2024-12-02T14:19:33,656 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:33,656 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/B is initiating minor compaction (all files) 2024-12-02T14:19:33,656 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/B in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:33,656 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/9db191b33da54b3da7e195a5f4e3110e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5794c44ad3ce42609955111f2c71eb29, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3d7b0fb8086346beb5726ffeac283d70, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/cfb013639aa942c2984365259859316b] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=44.6 K 2024-12-02T14:19:33,657 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 9db191b33da54b3da7e195a5f4e3110e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733149171678 2024-12-02T14:19:33,658 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 5794c44ad3ce42609955111f2c71eb29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733149171792 2024-12-02T14:19:33,658 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d7b0fb8086346beb5726ffeac283d70, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733149172455 2024-12-02T14:19:33,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:33,658 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting cfb013639aa942c2984365259859316b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733149172856 2024-12-02T14:19:33,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:33,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:33,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:33,678 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:33,681 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412021394e9235566432b9acd225c2399b50f_367203f21af3d1a942fdfc82bb3c32fd store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:33,684 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412021394e9235566432b9acd225c2399b50f_367203f21af3d1a942fdfc82bb3c32fd, store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:33,684 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#B#compaction#124 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:33,685 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021394e9235566432b9acd225c2399b50f_367203f21af3d1a942fdfc82bb3c32fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:33,685 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/57e80fa23547478b83d445d14ecd2b4b is 50, key is test_row_0/B:col10/1733149173472/Put/seqid=0 2024-12-02T14:19:33,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741967_1143 (size=12139) 2024-12-02T14:19:33,715 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/57e80fa23547478b83d445d14ecd2b4b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/57e80fa23547478b83d445d14ecd2b4b 2024-12-02T14:19:33,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741968_1144 (size=4469) 2024-12-02T14:19:33,726 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/B of 367203f21af3d1a942fdfc82bb3c32fd into 57e80fa23547478b83d445d14ecd2b4b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:33,726 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:33,726 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/B, priority=12, startTime=1733149173653; duration=0sec 2024-12-02T14:19:33,726 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:33,726 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:B 2024-12-02T14:19:33,726 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:33,727 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#A#compaction#123 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:33,728 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/0d20c8d11e4d4ad8b7eb1c5494613f01 is 175, key is test_row_0/A:col10/1733149173472/Put/seqid=0 2024-12-02T14:19:33,730 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:33,730 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/C is initiating minor compaction (all files) 2024-12-02T14:19:33,731 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/C in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:33,731 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4e7670f601284f028a0b2aec428a04d8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/9f8108063ca943d596af44827d3cad63, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/76b7e43c322048c889bf39f4435b175e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/b961a273a0c444dcbb5927f37a650178] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=44.6 K 2024-12-02T14:19:33,731 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e7670f601284f028a0b2aec428a04d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733149171678 2024-12-02T14:19:33,734 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f8108063ca943d596af44827d3cad63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733149171792 2024-12-02T14:19:33,735 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 76b7e43c322048c889bf39f4435b175e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733149172455 2024-12-02T14:19:33,735 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting b961a273a0c444dcbb5927f37a650178, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733149172856 2024-12-02T14:19:33,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741969_1145 (size=31093) 2024-12-02T14:19:33,757 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#C#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:33,758 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/d6c843450a264f69af121a15a9a7b764 is 50, key is test_row_0/C:col10/1733149173472/Put/seqid=0 2024-12-02T14:19:33,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741970_1146 (size=12139) 2024-12-02T14:19:33,790 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/d6c843450a264f69af121a15a9a7b764 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/d6c843450a264f69af121a15a9a7b764 2024-12-02T14:19:33,804 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/C of 367203f21af3d1a942fdfc82bb3c32fd into d6c843450a264f69af121a15a9a7b764(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:33,804 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:33,804 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/C, priority=12, startTime=1733149173659; duration=0sec 2024-12-02T14:19:33,804 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:33,804 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:C 2024-12-02T14:19:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T14:19:33,821 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:33,821 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-02T14:19:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T14:19:33,824 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:33,825 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:33,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:33,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T14:19:33,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:33,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:33,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:33,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:33,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:33,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:33,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:33,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022ad9c13c891b45a98245b7b355098b85_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149173510/Put/seqid=0 2024-12-02T14:19:33,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741971_1147 (size=14594) 2024-12-02T14:19:33,871 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:33,884 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022ad9c13c891b45a98245b7b355098b85_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022ad9c13c891b45a98245b7b355098b85_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:33,886 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/3b57eccb8f8849e3a52833e4b73798b1, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:33,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/3b57eccb8f8849e3a52833e4b73798b1 is 175, key is test_row_0/A:col10/1733149173510/Put/seqid=0 2024-12-02T14:19:33,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741972_1148 (size=39549) 2024-12-02T14:19:33,917 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/3b57eccb8f8849e3a52833e4b73798b1 2024-12-02T14:19:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T14:19:33,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/6095981bdd6743ee9ab32acc3ae5d74d is 50, key is test_row_0/B:col10/1733149173510/Put/seqid=0 2024-12-02T14:19:33,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149173905, max timestamp=1733149173937Num uniques 859; 2024-12-02T14:19:33,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149173905, max timestamp=1733149173937Num uniques 859; 2024-12-02T14:19:33,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:19:33,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:19:33,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149173905, max timestamp=1733149173937Num uniques 859; 2024-12-02T14:19:33,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:19:33,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741973_1149 (size=12001) 2024-12-02T14:19:33,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/6095981bdd6743ee9ab32acc3ae5d74d 2024-12-02T14:19:33,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/546b86a76d5d44ae9a224048832c8e10 is 50, key is test_row_0/C:col10/1733149173510/Put/seqid=0 2024-12-02T14:19:33,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T14:19:33,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:33,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:33,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:33,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:33,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:33,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741974_1150 (size=12001) 2024-12-02T14:19:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:34,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149234007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:34,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:34,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48354 deadline: 1733149234018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149234018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149234018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:34,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149234029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:34,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:34,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149234119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:34,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T14:19:34,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:34,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:34,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:34,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:34,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:34,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T14:19:34,161 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/0d20c8d11e4d4ad8b7eb1c5494613f01 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/0d20c8d11e4d4ad8b7eb1c5494613f01 2024-12-02T14:19:34,168 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/A of 367203f21af3d1a942fdfc82bb3c32fd into 0d20c8d11e4d4ad8b7eb1c5494613f01(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:34,168 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:34,168 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/A, priority=12, startTime=1733149173649; duration=0sec 2024-12-02T14:19:34,168 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:34,168 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:A 2024-12-02T14:19:34,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T14:19:34,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:34,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:34,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:34,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:34,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:34,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:34,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:34,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149234329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:34,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:34,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/546b86a76d5d44ae9a224048832c8e10 2024-12-02T14:19:34,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/3b57eccb8f8849e3a52833e4b73798b1 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/3b57eccb8f8849e3a52833e4b73798b1 2024-12-02T14:19:34,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/3b57eccb8f8849e3a52833e4b73798b1, entries=200, sequenceid=109, filesize=38.6 K 2024-12-02T14:19:34,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/6095981bdd6743ee9ab32acc3ae5d74d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/6095981bdd6743ee9ab32acc3ae5d74d 2024-12-02T14:19:34,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/6095981bdd6743ee9ab32acc3ae5d74d, entries=150, sequenceid=109, filesize=11.7 K 2024-12-02T14:19:34,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/546b86a76d5d44ae9a224048832c8e10 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/546b86a76d5d44ae9a224048832c8e10 2024-12-02T14:19:34,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/546b86a76d5d44ae9a224048832c8e10, entries=150, sequenceid=109, filesize=11.7 K 2024-12-02T14:19:34,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 367203f21af3d1a942fdfc82bb3c32fd in 564ms, sequenceid=109, compaction requested=false 2024-12-02T14:19:34,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:34,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T14:19:34,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:34,443 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T14:19:34,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:34,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:34,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:34,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:34,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:34,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:34,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T14:19:34,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202fddacaccc00f4723943c09d3adde6512_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149174005/Put/seqid=0 2024-12-02T14:19:34,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741975_1151 (size=12304) 2024-12-02T14:19:34,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:34,468 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202fddacaccc00f4723943c09d3adde6512_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fddacaccc00f4723943c09d3adde6512_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:34,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/6648dfba666e46fa828aa486c3f282c7, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:34,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/6648dfba666e46fa828aa486c3f282c7 is 175, key is test_row_0/A:col10/1733149174005/Put/seqid=0 2024-12-02T14:19:34,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741976_1152 (size=31105) 2024-12-02T14:19:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:34,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:34,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149234668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:34,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:34,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149234779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:34,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:34,886 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/6648dfba666e46fa828aa486c3f282c7 2024-12-02T14:19:34,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/373c530fe2d141a0b851d6dca95492c6 is 50, key is test_row_0/B:col10/1733149174005/Put/seqid=0 2024-12-02T14:19:34,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741977_1153 (size=12151) 2024-12-02T14:19:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T14:19:35,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149234995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:35,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149235029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:35,034 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 367203f21af3d1a942fdfc82bb3c32fd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:35,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48354 deadline: 1733149235039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:35,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149235039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:35,042 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 367203f21af3d1a942fdfc82bb3c32fd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:35,042 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 367203f21af3d1a942fdfc82bb3c32fd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:35,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:35,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:35,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:35,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:35,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:35,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:35,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149235048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:35,052 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_1' in 367203f21af3d1a942fdfc82bb3c32fd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:35,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:35,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:35,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:35,305 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/373c530fe2d141a0b851d6dca95492c6 2024-12-02T14:19:35,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/8f434b14a2884803a189b37626dade18 is 50, key is test_row_0/C:col10/1733149174005/Put/seqid=0 2024-12-02T14:19:35,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149235308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:35,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741978_1154 (size=12151) 2024-12-02T14:19:35,719 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/8f434b14a2884803a189b37626dade18 2024-12-02T14:19:35,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/6648dfba666e46fa828aa486c3f282c7 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/6648dfba666e46fa828aa486c3f282c7 2024-12-02T14:19:35,731 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/6648dfba666e46fa828aa486c3f282c7, entries=150, sequenceid=147, filesize=30.4 K 2024-12-02T14:19:35,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/373c530fe2d141a0b851d6dca95492c6 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/373c530fe2d141a0b851d6dca95492c6 2024-12-02T14:19:35,738 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/373c530fe2d141a0b851d6dca95492c6, entries=150, sequenceid=147, filesize=11.9 K 2024-12-02T14:19:35,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/8f434b14a2884803a189b37626dade18 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/8f434b14a2884803a189b37626dade18 2024-12-02T14:19:35,745 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/8f434b14a2884803a189b37626dade18, entries=150, sequenceid=147, filesize=11.9 K 2024-12-02T14:19:35,746 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 367203f21af3d1a942fdfc82bb3c32fd in 1302ms, sequenceid=147, compaction requested=true 2024-12-02T14:19:35,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:35,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:35,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-02T14:19:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-02T14:19:35,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-02T14:19:35,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9220 sec 2024-12-02T14:19:35,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.9290 sec 2024-12-02T14:19:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:35,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T14:19:35,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:35,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:35,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:35,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:35,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:35,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:35,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025fbab95f558b4313a234331a0c00f2d6_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149174660/Put/seqid=0 2024-12-02T14:19:35,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741979_1155 (size=14794) 2024-12-02T14:19:35,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149175889, max timestamp=1733149175909Num uniques 859; 2024-12-02T14:19:35,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:19:35,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149175889, max timestamp=1733149175909Num uniques 859; 2024-12-02T14:19:35,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:19:35,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149175889, max timestamp=1733149175909Num uniques 859; 2024-12-02T14:19:35,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:19:35,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149235965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:35,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:35,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T14:19:35,968 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:35,969 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-02T14:19:35,971 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T14:19:35,972 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:35,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T14:19:36,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149236078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:36,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:36,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T14:19:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:36,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,240 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:36,244 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025fbab95f558b4313a234331a0c00f2d6_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025fbab95f558b4313a234331a0c00f2d6_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:36,245 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/47fb727654a04f37ad10d8a44ee5f91c, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:36,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/47fb727654a04f37ad10d8a44ee5f91c is 175, key is test_row_0/A:col10/1733149174660/Put/seqid=0 2024-12-02T14:19:36,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741980_1156 (size=39749) 2024-12-02T14:19:36,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T14:19:36,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:36,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:36,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:36,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T14:19:36,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149236288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:36,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:36,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:36,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:36,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T14:19:36,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:36,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:36,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:36,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T14:19:36,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:36,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:36,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:36,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T14:19:36,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149236599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:36,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:36,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:36,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:36,651 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/47fb727654a04f37ad10d8a44ee5f91c 2024-12-02T14:19:36,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/5caef67c4c7e440696599b08c5348817 is 50, key is test_row_0/B:col10/1733149174660/Put/seqid=0 2024-12-02T14:19:36,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741981_1157 (size=12151) 2024-12-02T14:19:36,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T14:19:36,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:36,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:36,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:36,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T14:19:36,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:36,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:36,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:36,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:36,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:37,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T14:19:37,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:37,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:37,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:37,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:37,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:37,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:37,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149237048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:37,051 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 367203f21af3d1a942fdfc82bb3c32fd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:37,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:37,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:37,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:37,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149237058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:37,062 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 367203f21af3d1a942fdfc82bb3c32fd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:37,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/5caef67c4c7e440696599b08c5348817 2024-12-02T14:19:37,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/0fe722e35c3047f78e8b5ccf8c685300 is 50, key is test_row_0/C:col10/1733149174660/Put/seqid=0 2024-12-02T14:19:37,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741982_1158 (size=12151) 2024-12-02T14:19:37,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:37,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149237069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48354 deadline: 1733149237069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:37,076 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_1' in 367203f21af3d1a942fdfc82bb3c32fd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:37,076 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 367203f21af3d1a942fdfc82bb3c32fd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T14:19:37,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149237108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:37,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:37,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:37,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:37,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T14:19:37,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:37,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:37,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:37,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:37,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:37,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T14:19:37,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:37,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:37,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:37,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:37,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:37,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/0fe722e35c3047f78e8b5ccf8c685300 2024-12-02T14:19:37,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/47fb727654a04f37ad10d8a44ee5f91c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/47fb727654a04f37ad10d8a44ee5f91c 2024-12-02T14:19:37,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/47fb727654a04f37ad10d8a44ee5f91c, entries=200, sequenceid=158, filesize=38.8 K 2024-12-02T14:19:37,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/5caef67c4c7e440696599b08c5348817 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5caef67c4c7e440696599b08c5348817 2024-12-02T14:19:37,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5caef67c4c7e440696599b08c5348817, entries=150, sequenceid=158, filesize=11.9 K 2024-12-02T14:19:37,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/0fe722e35c3047f78e8b5ccf8c685300 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/0fe722e35c3047f78e8b5ccf8c685300 2024-12-02T14:19:37,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/0fe722e35c3047f78e8b5ccf8c685300, entries=150, sequenceid=158, filesize=11.9 K 2024-12-02T14:19:37,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 367203f21af3d1a942fdfc82bb3c32fd in 1679ms, sequenceid=158, compaction requested=true 2024-12-02T14:19:37,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:37,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:37,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:37,501 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:37,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:37,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:37,501 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:37,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:37,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:37,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T14:19:37,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:37,502 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-02T14:19:37,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:37,502 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141496 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:37,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:37,503 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/A is initiating minor compaction (all files) 2024-12-02T14:19:37,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:37,503 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/A in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:37,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:37,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:37,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:37,503 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/0d20c8d11e4d4ad8b7eb1c5494613f01, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/3b57eccb8f8849e3a52833e4b73798b1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/6648dfba666e46fa828aa486c3f282c7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/47fb727654a04f37ad10d8a44ee5f91c] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=138.2 K 2024-12-02T14:19:37,503 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:37,503 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/0d20c8d11e4d4ad8b7eb1c5494613f01, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/3b57eccb8f8849e3a52833e4b73798b1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/6648dfba666e46fa828aa486c3f282c7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/47fb727654a04f37ad10d8a44ee5f91c] 2024-12-02T14:19:37,503 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:37,503 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/B is initiating minor compaction (all files) 2024-12-02T14:19:37,503 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/B in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:37,503 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/57e80fa23547478b83d445d14ecd2b4b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/6095981bdd6743ee9ab32acc3ae5d74d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/373c530fe2d141a0b851d6dca95492c6, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5caef67c4c7e440696599b08c5348817] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=47.3 K 2024-12-02T14:19:37,504 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e80fa23547478b83d445d14ecd2b4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733149172856 2024-12-02T14:19:37,504 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d20c8d11e4d4ad8b7eb1c5494613f01, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733149172856 2024-12-02T14:19:37,504 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 6095981bdd6743ee9ab32acc3ae5d74d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733149173499 2024-12-02T14:19:37,504 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b57eccb8f8849e3a52833e4b73798b1, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733149173499 2024-12-02T14:19:37,504 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 373c530fe2d141a0b851d6dca95492c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733149173990 2024-12-02T14:19:37,504 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6648dfba666e46fa828aa486c3f282c7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733149173990 2024-12-02T14:19:37,505 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47fb727654a04f37ad10d8a44ee5f91c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733149174660 2024-12-02T14:19:37,505 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 5caef67c4c7e440696599b08c5348817, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733149174660 2024-12-02T14:19:37,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120219d920d6eb7c483fb8ca52289fb7d185_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149175956/Put/seqid=0 2024-12-02T14:19:37,514 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:37,516 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#B#compaction#137 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:37,517 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/3560ad536c5d4fbab0cbd7ef3cbe940a is 50, key is test_row_0/B:col10/1733149174660/Put/seqid=0 2024-12-02T14:19:37,517 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202944322b1b43a43009af9adadf34a2c7c_367203f21af3d1a942fdfc82bb3c32fd store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:37,520 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202944322b1b43a43009af9adadf34a2c7c_367203f21af3d1a942fdfc82bb3c32fd, store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:37,520 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202944322b1b43a43009af9adadf34a2c7c_367203f21af3d1a942fdfc82bb3c32fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:37,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741983_1159 (size=12304) 2024-12-02T14:19:37,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741985_1161 (size=12425) 2024-12-02T14:19:37,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741984_1160 (size=4469) 2024-12-02T14:19:37,534 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#A#compaction#136 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:37,535 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/dd7a9e0d93c64329ae1828789316c516 is 175, key is test_row_0/A:col10/1733149174660/Put/seqid=0 2024-12-02T14:19:37,539 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/3560ad536c5d4fbab0cbd7ef3cbe940a as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3560ad536c5d4fbab0cbd7ef3cbe940a 2024-12-02T14:19:37,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741986_1162 (size=31379) 2024-12-02T14:19:37,545 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/dd7a9e0d93c64329ae1828789316c516 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/dd7a9e0d93c64329ae1828789316c516 2024-12-02T14:19:37,546 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/B of 367203f21af3d1a942fdfc82bb3c32fd into 3560ad536c5d4fbab0cbd7ef3cbe940a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:37,546 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:37,546 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/B, priority=12, startTime=1733149177501; duration=0sec 2024-12-02T14:19:37,546 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:37,546 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:B 2024-12-02T14:19:37,546 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:37,551 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:37,551 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/C is initiating minor compaction (all files) 2024-12-02T14:19:37,551 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/C in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:37,552 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/d6c843450a264f69af121a15a9a7b764, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/546b86a76d5d44ae9a224048832c8e10, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/8f434b14a2884803a189b37626dade18, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/0fe722e35c3047f78e8b5ccf8c685300] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=47.3 K 2024-12-02T14:19:37,553 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting d6c843450a264f69af121a15a9a7b764, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733149172856 2024-12-02T14:19:37,554 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 546b86a76d5d44ae9a224048832c8e10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733149173499 2024-12-02T14:19:37,555 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f434b14a2884803a189b37626dade18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733149173990 2024-12-02T14:19:37,555 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/A of 367203f21af3d1a942fdfc82bb3c32fd into dd7a9e0d93c64329ae1828789316c516(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:37,555 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:37,555 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/A, priority=12, startTime=1733149177501; duration=0sec 2024-12-02T14:19:37,555 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fe722e35c3047f78e8b5ccf8c685300, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733149174660 2024-12-02T14:19:37,555 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:37,555 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:A 2024-12-02T14:19:37,578 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:37,578 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/bf2e2500a2d84084ab0d754d669b7105 is 50, key is test_row_0/C:col10/1733149174660/Put/seqid=0 2024-12-02T14:19:37,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741987_1163 (size=12425) 2024-12-02T14:19:37,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:37,931 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120219d920d6eb7c483fb8ca52289fb7d185_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120219d920d6eb7c483fb8ca52289fb7d185_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:37,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/d4a89fbc574f459c92bfe6c58348fcb8, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:37,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/d4a89fbc574f459c92bfe6c58348fcb8 is 175, key is test_row_0/A:col10/1733149175956/Put/seqid=0 2024-12-02T14:19:37,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741988_1164 (size=31105) 2024-12-02T14:19:38,016 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/bf2e2500a2d84084ab0d754d669b7105 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/bf2e2500a2d84084ab0d754d669b7105 2024-12-02T14:19:38,021 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/C of 367203f21af3d1a942fdfc82bb3c32fd into bf2e2500a2d84084ab0d754d669b7105(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:38,021 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:38,021 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/C, priority=12, startTime=1733149177501; duration=0sec 2024-12-02T14:19:38,022 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:38,022 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:C 2024-12-02T14:19:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T14:19:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:38,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:38,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149238165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:38,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:38,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:38,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:38,273 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-02T14:19:38,273 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-02T14:19:38,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149238279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:38,343 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/d4a89fbc574f459c92bfe6c58348fcb8 2024-12-02T14:19:38,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/fc16a84bfc604ee78acc8a61c5f578f2 is 50, key is test_row_0/B:col10/1733149175956/Put/seqid=0 2024-12-02T14:19:38,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741989_1165 (size=12151) 2024-12-02T14:19:38,358 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/fc16a84bfc604ee78acc8a61c5f578f2 2024-12-02T14:19:38,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/f06052c5b8044e0397381b7703af2e28 is 50, key is test_row_0/C:col10/1733149175956/Put/seqid=0 2024-12-02T14:19:38,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741990_1166 (size=12151) 2024-12-02T14:19:38,383 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/f06052c5b8044e0397381b7703af2e28 2024-12-02T14:19:38,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/d4a89fbc574f459c92bfe6c58348fcb8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/d4a89fbc574f459c92bfe6c58348fcb8 2024-12-02T14:19:38,404 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/d4a89fbc574f459c92bfe6c58348fcb8, entries=150, sequenceid=193, filesize=30.4 K 2024-12-02T14:19:38,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/fc16a84bfc604ee78acc8a61c5f578f2 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/fc16a84bfc604ee78acc8a61c5f578f2 2024-12-02T14:19:38,410 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/fc16a84bfc604ee78acc8a61c5f578f2, entries=150, sequenceid=193, filesize=11.9 K 2024-12-02T14:19:38,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/f06052c5b8044e0397381b7703af2e28 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/f06052c5b8044e0397381b7703af2e28 2024-12-02T14:19:38,419 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/f06052c5b8044e0397381b7703af2e28, entries=150, sequenceid=193, filesize=11.9 K 2024-12-02T14:19:38,421 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 367203f21af3d1a942fdfc82bb3c32fd in 918ms, sequenceid=193, compaction requested=false 2024-12-02T14:19:38,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:38,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:38,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-02T14:19:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-02T14:19:38,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-02T14:19:38,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4520 sec 2024-12-02T14:19:38,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.4580 sec 2024-12-02T14:19:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:38,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T14:19:38,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:38,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:38,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:38,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:38,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:38,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:38,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e352ecd6a87c4eb3969498b2e4c43a0a_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149178158/Put/seqid=0 2024-12-02T14:19:38,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741991_1167 (size=14794) 2024-12-02T14:19:38,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149238584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:38,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149238699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:38,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149238908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:38,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:38,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:38,922 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:38,927 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e352ecd6a87c4eb3969498b2e4c43a0a_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e352ecd6a87c4eb3969498b2e4c43a0a_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:38,928 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/e0e0779e1c40497581330f75b79ffbf8, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:38,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/e0e0779e1c40497581330f75b79ffbf8 is 175, key is test_row_0/A:col10/1733149178158/Put/seqid=0 2024-12-02T14:19:38,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741992_1168 (size=39749) 2024-12-02T14:19:39,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149239218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:39,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:39,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:39,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:39,336 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/e0e0779e1c40497581330f75b79ffbf8 2024-12-02T14:19:39,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/3d9fd29af0de4169a1c6ad1c03fad06a is 50, key is test_row_0/B:col10/1733149178158/Put/seqid=0 2024-12-02T14:19:39,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741993_1169 (size=12151) 2024-12-02T14:19:39,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/3d9fd29af0de4169a1c6ad1c03fad06a 2024-12-02T14:19:39,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/4db5124d3c964ae59d19251c9c4506be is 50, key is test_row_0/C:col10/1733149178158/Put/seqid=0 2024-12-02T14:19:39,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741994_1170 (size=12151) 2024-12-02T14:19:39,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149239729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:39,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:39,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:39,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:39,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/4db5124d3c964ae59d19251c9c4506be 2024-12-02T14:19:39,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/e0e0779e1c40497581330f75b79ffbf8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/e0e0779e1c40497581330f75b79ffbf8 2024-12-02T14:19:39,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/e0e0779e1c40497581330f75b79ffbf8, entries=200, sequenceid=208, filesize=38.8 K 2024-12-02T14:19:39,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/3d9fd29af0de4169a1c6ad1c03fad06a as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3d9fd29af0de4169a1c6ad1c03fad06a 2024-12-02T14:19:39,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3d9fd29af0de4169a1c6ad1c03fad06a, entries=150, sequenceid=208, filesize=11.9 K 2024-12-02T14:19:39,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/4db5124d3c964ae59d19251c9c4506be as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4db5124d3c964ae59d19251c9c4506be 2024-12-02T14:19:39,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4db5124d3c964ae59d19251c9c4506be, entries=150, sequenceid=208, filesize=11.9 K 2024-12-02T14:19:39,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 367203f21af3d1a942fdfc82bb3c32fd in 1331ms, sequenceid=208, compaction requested=true 2024-12-02T14:19:39,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:39,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:39,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:39,823 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:39,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:39,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:39,823 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:39,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:39,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:39,825 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:39,825 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:39,825 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/A is initiating minor compaction (all files) 2024-12-02T14:19:39,825 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/B is initiating minor compaction (all files) 2024-12-02T14:19:39,825 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/A in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:39,825 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/dd7a9e0d93c64329ae1828789316c516, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/d4a89fbc574f459c92bfe6c58348fcb8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/e0e0779e1c40497581330f75b79ffbf8] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=99.8 K 2024-12-02T14:19:39,825 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:39,825 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/dd7a9e0d93c64329ae1828789316c516, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/d4a89fbc574f459c92bfe6c58348fcb8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/e0e0779e1c40497581330f75b79ffbf8] 2024-12-02T14:19:39,825 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/B in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:39,825 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3560ad536c5d4fbab0cbd7ef3cbe940a, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/fc16a84bfc604ee78acc8a61c5f578f2, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3d9fd29af0de4169a1c6ad1c03fad06a] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=35.9 K 2024-12-02T14:19:39,826 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd7a9e0d93c64329ae1828789316c516, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733149174660 2024-12-02T14:19:39,826 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 3560ad536c5d4fbab0cbd7ef3cbe940a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733149174660 2024-12-02T14:19:39,826 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4a89fbc574f459c92bfe6c58348fcb8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733149175951 2024-12-02T14:19:39,826 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting fc16a84bfc604ee78acc8a61c5f578f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733149175951 2024-12-02T14:19:39,826 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0e0779e1c40497581330f75b79ffbf8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733149178156 2024-12-02T14:19:39,826 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d9fd29af0de4169a1c6ad1c03fad06a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733149178156 2024-12-02T14:19:39,848 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:39,854 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#B#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:39,855 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/5293c652a711404e8b783a3d1d3f328d is 50, key is test_row_0/B:col10/1733149178158/Put/seqid=0 2024-12-02T14:19:39,861 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120265e6c08b782d422bbeac76f6332235eb_367203f21af3d1a942fdfc82bb3c32fd store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:39,863 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120265e6c08b782d422bbeac76f6332235eb_367203f21af3d1a942fdfc82bb3c32fd, store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:39,863 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120265e6c08b782d422bbeac76f6332235eb_367203f21af3d1a942fdfc82bb3c32fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:39,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741995_1171 (size=12527) 2024-12-02T14:19:39,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741996_1172 (size=4469) 2024-12-02T14:19:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T14:19:40,129 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:40,129 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-02T14:19:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T14:19:40,131 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:40,131 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:40,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T14:19:40,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-02T14:19:40,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:40,284 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T14:19:40,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:40,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:40,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:40,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:40,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:40,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:40,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ec93a8639fa04cc78c69462b03776abc_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149178582/Put/seqid=0 2024-12-02T14:19:40,319 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#A#compaction#144 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:40,319 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/027cbdb6a1d0475d92b4675cc4039b84 is 175, key is test_row_0/A:col10/1733149178158/Put/seqid=0 2024-12-02T14:19:40,322 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/5293c652a711404e8b783a3d1d3f328d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5293c652a711404e8b783a3d1d3f328d 2024-12-02T14:19:40,334 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/B of 367203f21af3d1a942fdfc82bb3c32fd into 5293c652a711404e8b783a3d1d3f328d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:40,334 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:40,334 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/B, priority=13, startTime=1733149179823; duration=0sec 2024-12-02T14:19:40,334 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:40,334 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:B 2024-12-02T14:19:40,334 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:40,336 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:40,336 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/C is initiating minor compaction (all files) 2024-12-02T14:19:40,336 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/C in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:40,336 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/bf2e2500a2d84084ab0d754d669b7105, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/f06052c5b8044e0397381b7703af2e28, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4db5124d3c964ae59d19251c9c4506be] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=35.9 K 2024-12-02T14:19:40,339 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting bf2e2500a2d84084ab0d754d669b7105, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733149174660 2024-12-02T14:19:40,339 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting f06052c5b8044e0397381b7703af2e28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733149175951 2024-12-02T14:19:40,340 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 4db5124d3c964ae59d19251c9c4506be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733149178156 2024-12-02T14:19:40,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741997_1173 (size=12304) 2024-12-02T14:19:40,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741998_1174 (size=31481) 2024-12-02T14:19:40,372 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#C#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:40,372 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/2a38f3a379d54590976119b74308a91e is 50, key is test_row_0/C:col10/1733149178158/Put/seqid=0 2024-12-02T14:19:40,376 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/027cbdb6a1d0475d92b4675cc4039b84 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/027cbdb6a1d0475d92b4675cc4039b84 2024-12-02T14:19:40,383 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/A of 367203f21af3d1a942fdfc82bb3c32fd into 027cbdb6a1d0475d92b4675cc4039b84(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:40,383 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:40,383 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/A, priority=13, startTime=1733149179823; duration=0sec 2024-12-02T14:19:40,383 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:40,383 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:A 2024-12-02T14:19:40,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741999_1175 (size=12527) 2024-12-02T14:19:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T14:19:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:40,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T14:19:40,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:40,770 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ec93a8639fa04cc78c69462b03776abc_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ec93a8639fa04cc78c69462b03776abc_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:40,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/e719f0679c3c4122804ff5fedf9e3f93, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:40,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/e719f0679c3c4122804ff5fedf9e3f93 is 175, key is test_row_0/A:col10/1733149178582/Put/seqid=0 2024-12-02T14:19:40,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149240781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:40,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:40,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:40,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:40,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742000_1176 (size=31105) 2024-12-02T14:19:40,806 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=228, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/e719f0679c3c4122804ff5fedf9e3f93 2024-12-02T14:19:40,833 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/2a38f3a379d54590976119b74308a91e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/2a38f3a379d54590976119b74308a91e 2024-12-02T14:19:40,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/95f89958a15a434db2fb9137d16fae4f is 50, key is test_row_0/B:col10/1733149178582/Put/seqid=0 2024-12-02T14:19:40,844 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/C of 367203f21af3d1a942fdfc82bb3c32fd into 2a38f3a379d54590976119b74308a91e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:40,844 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:40,844 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/C, priority=13, startTime=1733149179823; duration=0sec 2024-12-02T14:19:40,844 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:40,844 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:C 2024-12-02T14:19:40,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742001_1177 (size=12151) 2024-12-02T14:19:40,857 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/95f89958a15a434db2fb9137d16fae4f 2024-12-02T14:19:40,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/049b6c5a3e7d458fab8d2f3c5f522efc is 50, key is test_row_0/C:col10/1733149178582/Put/seqid=0 2024-12-02T14:19:40,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149240888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:40,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:40,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:40,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:40,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742002_1178 (size=12151) 2024-12-02T14:19:41,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149241060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:41,063 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 367203f21af3d1a942fdfc82bb3c32fd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:41,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:41,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48354 deadline: 1733149241088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:41,089 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 367203f21af3d1a942fdfc82bb3c32fd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:41,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149241099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:41,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149241099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:41,103 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 367203f21af3d1a942fdfc82bb3c32fd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:41,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:41,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149241109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:41,115 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_1' in 367203f21af3d1a942fdfc82bb3c32fd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:41,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:41,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:41,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:41,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T14:19:41,310 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/049b6c5a3e7d458fab8d2f3c5f522efc 2024-12-02T14:19:41,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/e719f0679c3c4122804ff5fedf9e3f93 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/e719f0679c3c4122804ff5fedf9e3f93 2024-12-02T14:19:41,330 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/e719f0679c3c4122804ff5fedf9e3f93, entries=150, sequenceid=228, filesize=30.4 K 2024-12-02T14:19:41,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/95f89958a15a434db2fb9137d16fae4f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/95f89958a15a434db2fb9137d16fae4f 2024-12-02T14:19:41,339 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/95f89958a15a434db2fb9137d16fae4f, entries=150, sequenceid=228, filesize=11.9 K 2024-12-02T14:19:41,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/049b6c5a3e7d458fab8d2f3c5f522efc as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/049b6c5a3e7d458fab8d2f3c5f522efc 2024-12-02T14:19:41,348 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/049b6c5a3e7d458fab8d2f3c5f522efc, entries=150, sequenceid=228, filesize=11.9 K 2024-12-02T14:19:41,349 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 367203f21af3d1a942fdfc82bb3c32fd in 1065ms, sequenceid=228, compaction requested=false 2024-12-02T14:19:41,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:41,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:41,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-02T14:19:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-02T14:19:41,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-02T14:19:41,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2210 sec 2024-12-02T14:19:41,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.2270 sec 2024-12-02T14:19:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:41,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T14:19:41,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:41,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:41,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:41,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:41,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:41,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:41,446 INFO [master/a3a61c9ba14f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-02T14:19:41,446 INFO [master/a3a61c9ba14f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-02T14:19:41,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a6b0a8156cef43faa66bc9ef2843b328_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149181410/Put/seqid=0 2024-12-02T14:19:41,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742003_1179 (size=14794) 2024-12-02T14:19:41,515 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:41,521 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a6b0a8156cef43faa66bc9ef2843b328_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a6b0a8156cef43faa66bc9ef2843b328_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:41,522 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/295e5bd0dde7413b9114eb1836a795a0, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:41,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/295e5bd0dde7413b9114eb1836a795a0 is 175, key is test_row_0/A:col10/1733149181410/Put/seqid=0 2024-12-02T14:19:41,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742004_1180 (size=39749) 2024-12-02T14:19:41,547 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/295e5bd0dde7413b9114eb1836a795a0 2024-12-02T14:19:41,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149241549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:41,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/5d41b44339814cce938c7b7c67e57a2a is 50, key is test_row_0/B:col10/1733149181410/Put/seqid=0 2024-12-02T14:19:41,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742005_1181 (size=12151) 2024-12-02T14:19:41,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149241658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:41,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149241872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:41,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/5d41b44339814cce938c7b7c67e57a2a 2024-12-02T14:19:41,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/33d38bbd282e4746acdc3f3b25193c1e is 50, key is test_row_0/C:col10/1733149181410/Put/seqid=0 2024-12-02T14:19:41,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742006_1182 (size=12151) 2024-12-02T14:19:41,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/33d38bbd282e4746acdc3f3b25193c1e 2024-12-02T14:19:41,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/295e5bd0dde7413b9114eb1836a795a0 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/295e5bd0dde7413b9114eb1836a795a0 2024-12-02T14:19:41,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/295e5bd0dde7413b9114eb1836a795a0, entries=200, sequenceid=244, filesize=38.8 K 2024-12-02T14:19:41,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/5d41b44339814cce938c7b7c67e57a2a as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5d41b44339814cce938c7b7c67e57a2a 2024-12-02T14:19:41,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5d41b44339814cce938c7b7c67e57a2a, entries=150, sequenceid=244, filesize=11.9 K 2024-12-02T14:19:42,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/33d38bbd282e4746acdc3f3b25193c1e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/33d38bbd282e4746acdc3f3b25193c1e 2024-12-02T14:19:42,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/33d38bbd282e4746acdc3f3b25193c1e, entries=150, sequenceid=244, filesize=11.9 K 2024-12-02T14:19:42,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 367203f21af3d1a942fdfc82bb3c32fd in 596ms, sequenceid=244, compaction requested=true 2024-12-02T14:19:42,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:42,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:42,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:42,008 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:42,008 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:42,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:42,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:42,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:42,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:42,009 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:42,009 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:42,009 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/A is initiating minor compaction (all files) 2024-12-02T14:19:42,009 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/B is initiating minor compaction (all files) 2024-12-02T14:19:42,010 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/A in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:42,010 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/B in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:42,010 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/027cbdb6a1d0475d92b4675cc4039b84, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/e719f0679c3c4122804ff5fedf9e3f93, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/295e5bd0dde7413b9114eb1836a795a0] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=99.9 K 2024-12-02T14:19:42,010 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5293c652a711404e8b783a3d1d3f328d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/95f89958a15a434db2fb9137d16fae4f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5d41b44339814cce938c7b7c67e57a2a] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=36.0 K 2024-12-02T14:19:42,010 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:42,010 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/027cbdb6a1d0475d92b4675cc4039b84, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/e719f0679c3c4122804ff5fedf9e3f93, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/295e5bd0dde7413b9114eb1836a795a0] 2024-12-02T14:19:42,010 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 027cbdb6a1d0475d92b4675cc4039b84, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733149178156 2024-12-02T14:19:42,010 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 5293c652a711404e8b783a3d1d3f328d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733149178156 2024-12-02T14:19:42,010 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting e719f0679c3c4122804ff5fedf9e3f93, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733149178569 2024-12-02T14:19:42,010 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 95f89958a15a434db2fb9137d16fae4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733149178569 2024-12-02T14:19:42,011 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 295e5bd0dde7413b9114eb1836a795a0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733149180768 2024-12-02T14:19:42,011 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d41b44339814cce938c7b7c67e57a2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733149180774 2024-12-02T14:19:42,019 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:42,021 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202e29fd01b24404f4aab666168dfd9f908_367203f21af3d1a942fdfc82bb3c32fd store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:42,022 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#B#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:42,023 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/1ce9430306244a71a04681c81897b2c4 is 50, key is test_row_0/B:col10/1733149181410/Put/seqid=0 2024-12-02T14:19:42,024 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202e29fd01b24404f4aab666168dfd9f908_367203f21af3d1a942fdfc82bb3c32fd, store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:42,024 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e29fd01b24404f4aab666168dfd9f908_367203f21af3d1a942fdfc82bb3c32fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:42,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742007_1183 (size=12629) 2024-12-02T14:19:42,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742008_1184 (size=4469) 2024-12-02T14:19:42,059 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/1ce9430306244a71a04681c81897b2c4 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/1ce9430306244a71a04681c81897b2c4 2024-12-02T14:19:42,067 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/B of 367203f21af3d1a942fdfc82bb3c32fd into 1ce9430306244a71a04681c81897b2c4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:42,067 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:42,067 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/B, priority=13, startTime=1733149182008; duration=0sec 2024-12-02T14:19:42,068 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:42,068 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:B 2024-12-02T14:19:42,068 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:42,070 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:42,070 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/C is initiating minor compaction (all files) 2024-12-02T14:19:42,070 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/C in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:42,070 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/2a38f3a379d54590976119b74308a91e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/049b6c5a3e7d458fab8d2f3c5f522efc, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/33d38bbd282e4746acdc3f3b25193c1e] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=36.0 K 2024-12-02T14:19:42,071 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a38f3a379d54590976119b74308a91e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733149178156 2024-12-02T14:19:42,072 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 049b6c5a3e7d458fab8d2f3c5f522efc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733149178569 2024-12-02T14:19:42,072 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 33d38bbd282e4746acdc3f3b25193c1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733149180774 2024-12-02T14:19:42,103 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#C#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:42,104 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/a27c749a598547fb8b95072a74e28c6e is 50, key is test_row_0/C:col10/1733149181410/Put/seqid=0 2024-12-02T14:19:42,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742009_1185 (size=12629) 2024-12-02T14:19:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:42,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T14:19:42,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:42,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:42,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:42,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:42,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:42,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:42,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120254742a1b63e846d7b4be5e936b3773b5_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149181540/Put/seqid=0 2024-12-02T14:19:42,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742010_1186 (size=14944) 2024-12-02T14:19:42,218 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:42,223 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120254742a1b63e846d7b4be5e936b3773b5_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120254742a1b63e846d7b4be5e936b3773b5_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:42,224 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/4fc85bfe01c542229b1d1566da72fa89, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:42,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/4fc85bfe01c542229b1d1566da72fa89 is 175, key is test_row_0/A:col10/1733149181540/Put/seqid=0 2024-12-02T14:19:42,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149242232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:42,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742011_1187 (size=39899) 2024-12-02T14:19:42,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T14:19:42,278 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:42,279 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-02T14:19:42,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T14:19:42,281 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:42,281 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:42,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:42,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149242352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:42,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:42,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:42,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T14:19:42,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T14:19:42,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:42,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:42,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:42,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:42,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:42,453 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#A#compaction#153 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:42,453 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/9fd30e4bb79f4917a011599564d376c4 is 175, key is test_row_0/A:col10/1733149181410/Put/seqid=0 2024-12-02T14:19:42,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742012_1188 (size=31583) 2024-12-02T14:19:42,519 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/a27c749a598547fb8b95072a74e28c6e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/a27c749a598547fb8b95072a74e28c6e 2024-12-02T14:19:42,524 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/C of 367203f21af3d1a942fdfc82bb3c32fd into a27c749a598547fb8b95072a74e28c6e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:42,524 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:42,524 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/C, priority=13, startTime=1733149182008; duration=0sec 2024-12-02T14:19:42,524 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:42,524 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:C 2024-12-02T14:19:42,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149242558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:42,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T14:19:42,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:42,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:42,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:42,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:42,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T14:19:42,639 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/4fc85bfe01c542229b1d1566da72fa89 2024-12-02T14:19:42,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/e7cf5724fc114f3b928bf1f29275610a is 50, key is test_row_0/B:col10/1733149181540/Put/seqid=0 2024-12-02T14:19:42,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742013_1189 (size=12301) 2024-12-02T14:19:42,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T14:19:42,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:42,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:42,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:42,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:42,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:42,864 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/9fd30e4bb79f4917a011599564d376c4 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/9fd30e4bb79f4917a011599564d376c4 2024-12-02T14:19:42,871 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/A of 367203f21af3d1a942fdfc82bb3c32fd into 9fd30e4bb79f4917a011599564d376c4(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:42,871 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:42,871 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/A, priority=13, startTime=1733149182008; duration=0sec 2024-12-02T14:19:42,871 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:42,871 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:A 2024-12-02T14:19:42,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149242869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:42,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:42,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T14:19:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:42,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T14:19:43,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T14:19:43,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:43,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:43,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:43,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:43,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:43,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/e7cf5724fc114f3b928bf1f29275610a 2024-12-02T14:19:43,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/b38443a537514ed8a044009516f82143 is 50, key is test_row_0/C:col10/1733149181540/Put/seqid=0 2024-12-02T14:19:43,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742014_1190 (size=12301) 2024-12-02T14:19:43,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T14:19:43,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:43,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:43,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:43,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:43,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:43,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T14:19:43,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:43,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:43,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:43,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:43,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:43,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149243390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:43,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T14:19:43,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T14:19:43,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:43,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:43,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:43,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:43,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:43,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:43,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/b38443a537514ed8a044009516f82143 2024-12-02T14:19:43,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/4fc85bfe01c542229b1d1566da72fa89 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/4fc85bfe01c542229b1d1566da72fa89 2024-12-02T14:19:43,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/4fc85bfe01c542229b1d1566da72fa89, entries=200, sequenceid=265, filesize=39.0 K 2024-12-02T14:19:43,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/e7cf5724fc114f3b928bf1f29275610a as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/e7cf5724fc114f3b928bf1f29275610a 2024-12-02T14:19:43,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/e7cf5724fc114f3b928bf1f29275610a, entries=150, sequenceid=265, filesize=12.0 K 2024-12-02T14:19:43,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/b38443a537514ed8a044009516f82143 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/b38443a537514ed8a044009516f82143 2024-12-02T14:19:43,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/b38443a537514ed8a044009516f82143, entries=150, sequenceid=265, filesize=12.0 K 2024-12-02T14:19:43,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 367203f21af3d1a942fdfc82bb3c32fd in 1381ms, sequenceid=265, compaction requested=false 2024-12-02T14:19:43,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:43,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T14:19:43,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:43,678 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T14:19:43,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:43,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:43,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:43,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:43,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:43,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:43,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202783a56121474481e9e1514adddcf11ad_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149182224/Put/seqid=0 2024-12-02T14:19:43,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742015_1191 (size=12454) 2024-12-02T14:19:44,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:44,113 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202783a56121474481e9e1514adddcf11ad_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202783a56121474481e9e1514adddcf11ad_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/3ae87b65ef61443d9539c27592b01a65, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:44,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/3ae87b65ef61443d9539c27592b01a65 is 175, key is test_row_0/A:col10/1733149182224/Put/seqid=0 2024-12-02T14:19:44,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742016_1192 (size=31255) 2024-12-02T14:19:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:44,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T14:19:44,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:44,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149244470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:44,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:44,520 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/3ae87b65ef61443d9539c27592b01a65 2024-12-02T14:19:44,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/39eef14864a14dcb9dead80305215be3 is 50, key is test_row_0/B:col10/1733149182224/Put/seqid=0 2024-12-02T14:19:44,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742017_1193 (size=12301) 2024-12-02T14:19:44,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149244579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:44,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149244789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:44,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:44,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:44,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:44,935 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/39eef14864a14dcb9dead80305215be3 2024-12-02T14:19:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/c595e88703f84c8ca2e9baaf9e36285f is 50, key is test_row_0/C:col10/1733149182224/Put/seqid=0 2024-12-02T14:19:44,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742018_1194 (size=12301) 2024-12-02T14:19:44,956 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/c595e88703f84c8ca2e9baaf9e36285f 2024-12-02T14:19:44,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/3ae87b65ef61443d9539c27592b01a65 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/3ae87b65ef61443d9539c27592b01a65 2024-12-02T14:19:44,970 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/3ae87b65ef61443d9539c27592b01a65, entries=150, sequenceid=279, filesize=30.5 K 2024-12-02T14:19:44,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/39eef14864a14dcb9dead80305215be3 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/39eef14864a14dcb9dead80305215be3 2024-12-02T14:19:44,976 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/39eef14864a14dcb9dead80305215be3, entries=150, sequenceid=279, filesize=12.0 K 2024-12-02T14:19:44,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/c595e88703f84c8ca2e9baaf9e36285f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/c595e88703f84c8ca2e9baaf9e36285f 2024-12-02T14:19:44,981 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/c595e88703f84c8ca2e9baaf9e36285f, entries=150, sequenceid=279, filesize=12.0 K 2024-12-02T14:19:44,982 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 367203f21af3d1a942fdfc82bb3c32fd in 1304ms, sequenceid=279, compaction requested=true 2024-12-02T14:19:44,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:44,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:44,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-02T14:19:44,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-02T14:19:44,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-02T14:19:44,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7050 sec 2024-12-02T14:19:44,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.7110 sec 2024-12-02T14:19:45,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T14:19:45,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:45,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:45,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:45,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:45,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:45,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:45,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202de66df25216b4a6887fb2428dd8a2eb8_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149184435/Put/seqid=0 2024-12-02T14:19:45,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742019_1195 (size=14994) 2024-12-02T14:19:45,150 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:45,157 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202de66df25216b4a6887fb2428dd8a2eb8_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202de66df25216b4a6887fb2428dd8a2eb8_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:45,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:45,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149245152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:45,160 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/53130dfd354a4427b25292ee5c613791, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:45,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/53130dfd354a4427b25292ee5c613791 is 175, key is test_row_0/A:col10/1733149184435/Put/seqid=0 2024-12-02T14:19:45,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742020_1196 (size=39949) 2024-12-02T14:19:45,187 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/53130dfd354a4427b25292ee5c613791 2024-12-02T14:19:45,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/219bd5b0b2c649a7a84f18a28054c223 is 50, key is test_row_0/B:col10/1733149184435/Put/seqid=0 2024-12-02T14:19:45,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742021_1197 (size=12301) 2024-12-02T14:19:45,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/219bd5b0b2c649a7a84f18a28054c223 2024-12-02T14:19:45,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/bb71b353337948ae954e1b756e9ee2ee is 50, key is test_row_0/C:col10/1733149184435/Put/seqid=0 2024-12-02T14:19:45,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742022_1198 (size=12301) 2024-12-02T14:19:45,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/bb71b353337948ae954e1b756e9ee2ee 2024-12-02T14:19:45,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149245271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:45,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/53130dfd354a4427b25292ee5c613791 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/53130dfd354a4427b25292ee5c613791 2024-12-02T14:19:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:45,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/53130dfd354a4427b25292ee5c613791, entries=200, sequenceid=300, filesize=39.0 K 2024-12-02T14:19:45,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/219bd5b0b2c649a7a84f18a28054c223 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/219bd5b0b2c649a7a84f18a28054c223 2024-12-02T14:19:45,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/219bd5b0b2c649a7a84f18a28054c223, entries=150, sequenceid=300, filesize=12.0 K 2024-12-02T14:19:45,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/bb71b353337948ae954e1b756e9ee2ee as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/bb71b353337948ae954e1b756e9ee2ee 2024-12-02T14:19:45,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/bb71b353337948ae954e1b756e9ee2ee, entries=150, sequenceid=300, filesize=12.0 K 2024-12-02T14:19:45,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 367203f21af3d1a942fdfc82bb3c32fd in 215ms, sequenceid=300, compaction requested=true 2024-12-02T14:19:45,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:45,318 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:45,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:45,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:45,318 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:45,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:45,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:45,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:45,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:45,322 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:45,322 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/B is initiating minor compaction (all files) 2024-12-02T14:19:45,322 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/B in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:45,323 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/1ce9430306244a71a04681c81897b2c4, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/e7cf5724fc114f3b928bf1f29275610a, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/39eef14864a14dcb9dead80305215be3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/219bd5b0b2c649a7a84f18a28054c223] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=48.4 K 2024-12-02T14:19:45,323 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142686 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:45,323 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/A is initiating minor compaction (all files) 2024-12-02T14:19:45,323 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/A in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:45,324 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/9fd30e4bb79f4917a011599564d376c4, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/4fc85bfe01c542229b1d1566da72fa89, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/3ae87b65ef61443d9539c27592b01a65, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/53130dfd354a4427b25292ee5c613791] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=139.3 K 2024-12-02T14:19:45,324 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:45,324 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/9fd30e4bb79f4917a011599564d376c4, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/4fc85bfe01c542229b1d1566da72fa89, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/3ae87b65ef61443d9539c27592b01a65, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/53130dfd354a4427b25292ee5c613791] 2024-12-02T14:19:45,324 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ce9430306244a71a04681c81897b2c4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733149180774 2024-12-02T14:19:45,325 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fd30e4bb79f4917a011599564d376c4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733149180774 2024-12-02T14:19:45,325 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting e7cf5724fc114f3b928bf1f29275610a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733149181540 2024-12-02T14:19:45,326 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fc85bfe01c542229b1d1566da72fa89, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733149181489 2024-12-02T14:19:45,326 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 39eef14864a14dcb9dead80305215be3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733149182219 2024-12-02T14:19:45,326 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ae87b65ef61443d9539c27592b01a65, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733149182219 2024-12-02T14:19:45,326 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 219bd5b0b2c649a7a84f18a28054c223, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733149184435 2024-12-02T14:19:45,327 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53130dfd354a4427b25292ee5c613791, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733149184435 2024-12-02T14:19:45,347 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#B#compaction#165 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:45,348 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/20a619bc0f1242609e3ba0199b476a87 is 50, key is test_row_0/B:col10/1733149184435/Put/seqid=0 2024-12-02T14:19:45,358 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:45,379 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202180af2be13de4603a48a433736e7237d_367203f21af3d1a942fdfc82bb3c32fd store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:45,381 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202180af2be13de4603a48a433736e7237d_367203f21af3d1a942fdfc82bb3c32fd, store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:45,381 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202180af2be13de4603a48a433736e7237d_367203f21af3d1a942fdfc82bb3c32fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:45,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742023_1199 (size=12915) 2024-12-02T14:19:45,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742024_1200 (size=4469) 2024-12-02T14:19:45,435 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#A#compaction#166 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:45,436 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/6557d1bb3a6d412eb6314560d8496392 is 175, key is test_row_0/A:col10/1733149184435/Put/seqid=0 2024-12-02T14:19:45,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742025_1201 (size=31869) 2024-12-02T14:19:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:45,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T14:19:45,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:45,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:45,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:45,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:45,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:45,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:45,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120251c2db7e2f894e6090d45f14446005c0_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149185479/Put/seqid=0 2024-12-02T14:19:45,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742026_1202 (size=14994) 2024-12-02T14:19:45,534 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:45,539 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120251c2db7e2f894e6090d45f14446005c0_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120251c2db7e2f894e6090d45f14446005c0_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:45,540 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/0b6d8b06d2b4414b8ce632719b3a5b99, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:45,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/0b6d8b06d2b4414b8ce632719b3a5b99 is 175, key is test_row_0/A:col10/1733149185479/Put/seqid=0 2024-12-02T14:19:45,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742027_1203 (size=39949) 2024-12-02T14:19:45,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149245591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:45,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149245708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:45,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:45,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:45,814 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/20a619bc0f1242609e3ba0199b476a87 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/20a619bc0f1242609e3ba0199b476a87 2024-12-02T14:19:45,818 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/B of 367203f21af3d1a942fdfc82bb3c32fd into 20a619bc0f1242609e3ba0199b476a87(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:45,818 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:45,818 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/B, priority=12, startTime=1733149185318; duration=0sec 2024-12-02T14:19:45,818 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:45,818 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:B 2024-12-02T14:19:45,818 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:19:45,820 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:19:45,820 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/C is initiating minor compaction (all files) 2024-12-02T14:19:45,820 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/C in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:45,820 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/a27c749a598547fb8b95072a74e28c6e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/b38443a537514ed8a044009516f82143, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/c595e88703f84c8ca2e9baaf9e36285f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/bb71b353337948ae954e1b756e9ee2ee] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=48.4 K 2024-12-02T14:19:45,820 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting a27c749a598547fb8b95072a74e28c6e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733149180774 2024-12-02T14:19:45,821 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting b38443a537514ed8a044009516f82143, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733149181540 2024-12-02T14:19:45,821 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting c595e88703f84c8ca2e9baaf9e36285f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733149182219 2024-12-02T14:19:45,821 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting bb71b353337948ae954e1b756e9ee2ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733149184435 2024-12-02T14:19:45,830 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#C#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:45,831 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/9366c1f0d0e8448cb9b7b67770d67f00 is 50, key is test_row_0/C:col10/1733149184435/Put/seqid=0 2024-12-02T14:19:45,857 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/6557d1bb3a6d412eb6314560d8496392 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/6557d1bb3a6d412eb6314560d8496392 2024-12-02T14:19:45,867 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/A of 367203f21af3d1a942fdfc82bb3c32fd into 6557d1bb3a6d412eb6314560d8496392(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:45,867 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:45,867 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/A, priority=12, startTime=1733149185318; duration=0sec 2024-12-02T14:19:45,868 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:45,868 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:A 2024-12-02T14:19:45,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742028_1204 (size=12915) 2024-12-02T14:19:45,898 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/9366c1f0d0e8448cb9b7b67770d67f00 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/9366c1f0d0e8448cb9b7b67770d67f00 2024-12-02T14:19:45,904 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/C of 367203f21af3d1a942fdfc82bb3c32fd into 9366c1f0d0e8448cb9b7b67770d67f00(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:45,904 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:45,905 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/C, priority=12, startTime=1733149185319; duration=0sec 2024-12-02T14:19:45,905 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:45,905 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:C 2024-12-02T14:19:45,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149245928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:45,971 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/0b6d8b06d2b4414b8ce632719b3a5b99 2024-12-02T14:19:45,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/05d78a8c437e4cd1b8af024c3450f010 is 50, key is test_row_0/B:col10/1733149185479/Put/seqid=0 2024-12-02T14:19:46,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742029_1205 (size=12301) 2024-12-02T14:19:46,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/05d78a8c437e4cd1b8af024c3450f010 2024-12-02T14:19:46,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/e04b2251b90447bebd71e0805f605a5e is 50, key is test_row_0/C:col10/1733149185479/Put/seqid=0 2024-12-02T14:19:46,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742030_1206 (size=12301) 2024-12-02T14:19:46,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/e04b2251b90447bebd71e0805f605a5e 2024-12-02T14:19:46,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/0b6d8b06d2b4414b8ce632719b3a5b99 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/0b6d8b06d2b4414b8ce632719b3a5b99 2024-12-02T14:19:46,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/0b6d8b06d2b4414b8ce632719b3a5b99, entries=200, sequenceid=312, filesize=39.0 K 2024-12-02T14:19:46,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/05d78a8c437e4cd1b8af024c3450f010 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/05d78a8c437e4cd1b8af024c3450f010 2024-12-02T14:19:46,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/05d78a8c437e4cd1b8af024c3450f010, entries=150, sequenceid=312, filesize=12.0 K 2024-12-02T14:19:46,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/e04b2251b90447bebd71e0805f605a5e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/e04b2251b90447bebd71e0805f605a5e 2024-12-02T14:19:46,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/e04b2251b90447bebd71e0805f605a5e, entries=150, sequenceid=312, filesize=12.0 K 2024-12-02T14:19:46,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 367203f21af3d1a942fdfc82bb3c32fd in 606ms, sequenceid=312, compaction requested=false 2024-12-02T14:19:46,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:46,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149185564, max timestamp=1733149186246Num uniques 859; 2024-12-02T14:19:46,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:19:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:46,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-12-02T14:19:46,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:46,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:46,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:46,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:46,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:46,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:19:46,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f56bad6d4dd84b0a94e2a9fda9636cb6_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149185564/Put/seqid=0 2024-12-02T14:19:46,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742031_1207 (size=14994) 2024-12-02T14:19:46,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149246315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:46,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149246429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T14:19:46,440 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:46,449 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-02T14:19:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T14:19:46,454 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:46,454 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:46,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T14:19:46,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T14:19:46,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:46,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:46,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:46,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:46,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:46,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149246639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:46,687 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:46,692 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f56bad6d4dd84b0a94e2a9fda9636cb6_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f56bad6d4dd84b0a94e2a9fda9636cb6_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:46,694 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/5e19e6c9c9954c068056ddc06fdb89b0, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:46,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/5e19e6c9c9954c068056ddc06fdb89b0 is 175, key is test_row_0/A:col10/1733149185564/Put/seqid=0 2024-12-02T14:19:46,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742032_1208 (size=39949) 2024-12-02T14:19:46,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T14:19:46,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:46,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:46,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:46,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:46,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T14:19:46,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T14:19:46,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:46,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:46,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:46,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:46,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:46,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:46,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:46,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149246949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:46,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:46,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:46,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:47,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T14:19:47,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:47,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:47,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:47,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:47,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T14:19:47,120 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/5e19e6c9c9954c068056ddc06fdb89b0 2024-12-02T14:19:47,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/2bc3f68d49464064b73b0e627c4c27cf is 50, key is test_row_0/B:col10/1733149185564/Put/seqid=0 2024-12-02T14:19:47,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742033_1209 (size=12301) 2024-12-02T14:19:47,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/2bc3f68d49464064b73b0e627c4c27cf 2024-12-02T14:19:47,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/4dd9dc20a0814401904d66a1ca49e7a4 is 50, key is test_row_0/C:col10/1733149185564/Put/seqid=0 2024-12-02T14:19:47,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742034_1210 (size=12301) 2024-12-02T14:19:47,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T14:19:47,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:47,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:47,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:47,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:47,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:47,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T14:19:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:47,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:47,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:47,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149247469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:47,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T14:19:47,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:47,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:47,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:47,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:47,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:47,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/4dd9dc20a0814401904d66a1ca49e7a4 2024-12-02T14:19:47,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/5e19e6c9c9954c068056ddc06fdb89b0 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/5e19e6c9c9954c068056ddc06fdb89b0 2024-12-02T14:19:47,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/5e19e6c9c9954c068056ddc06fdb89b0, entries=200, sequenceid=336, filesize=39.0 K 2024-12-02T14:19:47,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/2bc3f68d49464064b73b0e627c4c27cf as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/2bc3f68d49464064b73b0e627c4c27cf 2024-12-02T14:19:47,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/2bc3f68d49464064b73b0e627c4c27cf, entries=150, sequenceid=336, filesize=12.0 K 2024-12-02T14:19:47,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/4dd9dc20a0814401904d66a1ca49e7a4 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4dd9dc20a0814401904d66a1ca49e7a4 2024-12-02T14:19:47,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4dd9dc20a0814401904d66a1ca49e7a4, entries=150, sequenceid=336, filesize=12.0 K 2024-12-02T14:19:47,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 367203f21af3d1a942fdfc82bb3c32fd in 1315ms, sequenceid=336, compaction requested=true 2024-12-02T14:19:47,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:47,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:47,565 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:47,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:47,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:47,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:47,565 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:47,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:47,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:47,567 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:47,567 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111767 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:47,567 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/B is initiating minor compaction (all files) 2024-12-02T14:19:47,567 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/A is initiating minor compaction (all files) 2024-12-02T14:19:47,568 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/A in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:47,568 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/B in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:47,568 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/20a619bc0f1242609e3ba0199b476a87, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/05d78a8c437e4cd1b8af024c3450f010, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/2bc3f68d49464064b73b0e627c4c27cf] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=36.6 K 2024-12-02T14:19:47,568 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/6557d1bb3a6d412eb6314560d8496392, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/0b6d8b06d2b4414b8ce632719b3a5b99, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/5e19e6c9c9954c068056ddc06fdb89b0] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=109.1 K 2024-12-02T14:19:47,568 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:47,568 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/6557d1bb3a6d412eb6314560d8496392, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/0b6d8b06d2b4414b8ce632719b3a5b99, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/5e19e6c9c9954c068056ddc06fdb89b0] 2024-12-02T14:19:47,568 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 20a619bc0f1242609e3ba0199b476a87, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733149184435 2024-12-02T14:19:47,568 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6557d1bb3a6d412eb6314560d8496392, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733149184435 2024-12-02T14:19:47,568 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 05d78a8c437e4cd1b8af024c3450f010, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733149185133 2024-12-02T14:19:47,568 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b6d8b06d2b4414b8ce632719b3a5b99, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733149185133 2024-12-02T14:19:47,569 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bc3f68d49464064b73b0e627c4c27cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733149185564 2024-12-02T14:19:47,569 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e19e6c9c9954c068056ddc06fdb89b0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733149185564 2024-12-02T14:19:47,576 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:47,577 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#B#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:47,578 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/cafdf05a0b154f82ba837f52f01fa1e7 is 50, key is test_row_0/B:col10/1733149185564/Put/seqid=0 2024-12-02T14:19:47,578 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202a6ea8422b07d4c739eecc321905435db_367203f21af3d1a942fdfc82bb3c32fd store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:47,580 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202a6ea8422b07d4c739eecc321905435db_367203f21af3d1a942fdfc82bb3c32fd, store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:47,581 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a6ea8422b07d4c739eecc321905435db_367203f21af3d1a942fdfc82bb3c32fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T14:19:47,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742035_1211 (size=13017) 2024-12-02T14:19:47,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742036_1212 (size=4469) 2024-12-02T14:19:47,601 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/cafdf05a0b154f82ba837f52f01fa1e7 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/cafdf05a0b154f82ba837f52f01fa1e7 2024-12-02T14:19:47,601 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#A#compaction#175 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:47,602 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/9102eec7b24248deb1902bdf1d35d874 is 175, key is test_row_0/A:col10/1733149185564/Put/seqid=0 2024-12-02T14:19:47,608 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/B of 367203f21af3d1a942fdfc82bb3c32fd into cafdf05a0b154f82ba837f52f01fa1e7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:47,608 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:47,608 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/B, priority=13, startTime=1733149187565; duration=0sec 2024-12-02T14:19:47,608 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:47,608 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:B 2024-12-02T14:19:47,608 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:47,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742037_1213 (size=31971) 2024-12-02T14:19:47,610 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:47,610 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/C is initiating minor compaction (all files) 2024-12-02T14:19:47,611 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/C in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:47,611 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/9366c1f0d0e8448cb9b7b67770d67f00, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/e04b2251b90447bebd71e0805f605a5e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4dd9dc20a0814401904d66a1ca49e7a4] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=36.6 K 2024-12-02T14:19:47,611 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 9366c1f0d0e8448cb9b7b67770d67f00, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733149184435 2024-12-02T14:19:47,611 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting e04b2251b90447bebd71e0805f605a5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733149185133 2024-12-02T14:19:47,612 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dd9dc20a0814401904d66a1ca49e7a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733149185564 2024-12-02T14:19:47,621 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#C#compaction#176 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:47,622 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/bfbffa36591b4bfcb5bd3c02b05bce9f is 50, key is test_row_0/C:col10/1733149185564/Put/seqid=0 2024-12-02T14:19:47,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742038_1214 (size=13017) 2024-12-02T14:19:47,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T14:19:47,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:47,681 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T14:19:47,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:47,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:47,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:47,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:47,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:47,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:47,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e3407f926ec2492bb23c44caf5af8ce1_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149186313/Put/seqid=0 2024-12-02T14:19:47,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742039_1215 (size=12454) 2024-12-02T14:19:48,016 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/9102eec7b24248deb1902bdf1d35d874 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/9102eec7b24248deb1902bdf1d35d874 2024-12-02T14:19:48,021 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/A of 367203f21af3d1a942fdfc82bb3c32fd into 9102eec7b24248deb1902bdf1d35d874(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:48,021 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:48,021 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/A, priority=13, startTime=1733149187565; duration=0sec 2024-12-02T14:19:48,021 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:48,021 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:A 2024-12-02T14:19:48,042 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/bfbffa36591b4bfcb5bd3c02b05bce9f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/bfbffa36591b4bfcb5bd3c02b05bce9f 2024-12-02T14:19:48,048 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/C of 367203f21af3d1a942fdfc82bb3c32fd into bfbffa36591b4bfcb5bd3c02b05bce9f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:48,048 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:48,048 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/C, priority=13, startTime=1733149187565; duration=0sec 2024-12-02T14:19:48,048 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:48,048 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:C 2024-12-02T14:19:48,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:48,112 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e3407f926ec2492bb23c44caf5af8ce1_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e3407f926ec2492bb23c44caf5af8ce1_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:48,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/77d2f651334342f3be811733747dc29b, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:48,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/77d2f651334342f3be811733747dc29b is 175, key is test_row_0/A:col10/1733149186313/Put/seqid=0 2024-12-02T14:19:48,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742040_1216 (size=31255) 2024-12-02T14:19:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:48,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:48,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149248507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:48,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:48,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:48,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:48,518 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/77d2f651334342f3be811733747dc29b 2024-12-02T14:19:48,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/18ddaf7cacb34b3fb83f12bece75a12c is 50, key is test_row_0/B:col10/1733149186313/Put/seqid=0 2024-12-02T14:19:48,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742041_1217 (size=12301) 2024-12-02T14:19:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T14:19:48,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149248618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:48,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:48,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:48,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:48,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149248828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:48,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:48,931 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/18ddaf7cacb34b3fb83f12bece75a12c 2024-12-02T14:19:48,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/8bc24a4c86d44a31a36e2ac24fa04532 is 50, key is test_row_0/C:col10/1733149186313/Put/seqid=0 2024-12-02T14:19:48,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742042_1218 (size=12301) 2024-12-02T14:19:49,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:49,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149249138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:49,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:49,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:49,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:49,343 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/8bc24a4c86d44a31a36e2ac24fa04532 2024-12-02T14:19:49,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/77d2f651334342f3be811733747dc29b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/77d2f651334342f3be811733747dc29b 2024-12-02T14:19:49,352 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/77d2f651334342f3be811733747dc29b, entries=150, sequenceid=353, filesize=30.5 K 2024-12-02T14:19:49,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/18ddaf7cacb34b3fb83f12bece75a12c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/18ddaf7cacb34b3fb83f12bece75a12c 2024-12-02T14:19:49,357 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/18ddaf7cacb34b3fb83f12bece75a12c, entries=150, sequenceid=353, filesize=12.0 K 2024-12-02T14:19:49,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/8bc24a4c86d44a31a36e2ac24fa04532 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/8bc24a4c86d44a31a36e2ac24fa04532 2024-12-02T14:19:49,363 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/8bc24a4c86d44a31a36e2ac24fa04532, entries=150, sequenceid=353, filesize=12.0 K 2024-12-02T14:19:49,365 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 367203f21af3d1a942fdfc82bb3c32fd in 1684ms, sequenceid=353, compaction requested=false 2024-12-02T14:19:49,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:49,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:49,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-02T14:19:49,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-02T14:19:49,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-02T14:19:49,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9120 sec 2024-12-02T14:19:49,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.9200 sec 2024-12-02T14:19:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:49,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T14:19:49,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:49,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:49,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:49,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:49,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:49,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:49,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d9931f30af584c15aa92e06d355fe7be_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149188506/Put/seqid=0 2024-12-02T14:19:49,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742043_1219 (size=14994) 2024-12-02T14:19:49,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149249688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:49,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:49,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:49,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:49,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149249798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:49,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:49,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:49,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:50,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149250008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:50,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:50,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:50,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:50,062 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:50,065 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d9931f30af584c15aa92e06d355fe7be_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d9931f30af584c15aa92e06d355fe7be_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:50,066 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/33c7a257e7064573845d10ee966f55b1, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:50,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/33c7a257e7064573845d10ee966f55b1 is 175, key is test_row_0/A:col10/1733149188506/Put/seqid=0 2024-12-02T14:19:50,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742044_1220 (size=39949) 2024-12-02T14:19:50,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:50,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149250318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:50,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:50,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:50,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:50,476 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/33c7a257e7064573845d10ee966f55b1 2024-12-02T14:19:50,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/2f2d6c130e7547208ff5de143838a94f is 50, key is test_row_0/B:col10/1733149188506/Put/seqid=0 2024-12-02T14:19:50,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742045_1221 (size=12301) 2024-12-02T14:19:50,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/2f2d6c130e7547208ff5de143838a94f 2024-12-02T14:19:50,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/4c21eb513ce6457fbbee92c06c3d4c0d is 50, key is test_row_0/C:col10/1733149188506/Put/seqid=0 2024-12-02T14:19:50,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742046_1222 (size=12301) 2024-12-02T14:19:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T14:19:50,609 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:50,609 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:19:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-02T14:19:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T14:19:50,611 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:19:50,611 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:19:50,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:19:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T14:19:50,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T14:19:50,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:50,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:50,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:50,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:50,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:50,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149250828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:50,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:50,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/4c21eb513ce6457fbbee92c06c3d4c0d 2024-12-02T14:19:50,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/33c7a257e7064573845d10ee966f55b1 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/33c7a257e7064573845d10ee966f55b1 2024-12-02T14:19:50,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/33c7a257e7064573845d10ee966f55b1, entries=200, sequenceid=372, filesize=39.0 K 2024-12-02T14:19:50,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/2f2d6c130e7547208ff5de143838a94f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/2f2d6c130e7547208ff5de143838a94f 2024-12-02T14:19:50,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/2f2d6c130e7547208ff5de143838a94f, entries=150, sequenceid=372, filesize=12.0 K 2024-12-02T14:19:50,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/4c21eb513ce6457fbbee92c06c3d4c0d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4c21eb513ce6457fbbee92c06c3d4c0d 2024-12-02T14:19:50,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T14:19:50,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:50,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:50,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:50,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:50,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:50,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4c21eb513ce6457fbbee92c06c3d4c0d, entries=150, sequenceid=372, filesize=12.0 K 2024-12-02T14:19:50,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 367203f21af3d1a942fdfc82bb3c32fd in 1269ms, sequenceid=372, compaction requested=true 2024-12-02T14:19:50,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:50,918 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:50,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:50,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:50,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:50,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:50,918 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:50,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:50,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:19:50,920 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:50,920 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/A is initiating minor compaction (all files) 2024-12-02T14:19:50,920 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/A in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:50,920 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/9102eec7b24248deb1902bdf1d35d874, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/77d2f651334342f3be811733747dc29b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/33c7a257e7064573845d10ee966f55b1] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=100.8 K 2024-12-02T14:19:50,920 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:50,920 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/9102eec7b24248deb1902bdf1d35d874, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/77d2f651334342f3be811733747dc29b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/33c7a257e7064573845d10ee966f55b1] 2024-12-02T14:19:50,920 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:50,921 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9102eec7b24248deb1902bdf1d35d874, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733149185564 2024-12-02T14:19:50,921 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/B is initiating minor compaction (all files) 2024-12-02T14:19:50,921 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/B in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:50,921 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/cafdf05a0b154f82ba837f52f01fa1e7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/18ddaf7cacb34b3fb83f12bece75a12c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/2f2d6c130e7547208ff5de143838a94f] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=36.7 K 2024-12-02T14:19:50,921 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77d2f651334342f3be811733747dc29b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733149186299 2024-12-02T14:19:50,921 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting cafdf05a0b154f82ba837f52f01fa1e7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733149185564 2024-12-02T14:19:50,922 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33c7a257e7064573845d10ee966f55b1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733149188497 2024-12-02T14:19:50,922 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 18ddaf7cacb34b3fb83f12bece75a12c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733149186299 2024-12-02T14:19:50,922 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f2d6c130e7547208ff5de143838a94f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733149188502 2024-12-02T14:19:50,927 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T14:19:50,938 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#B#compaction#184 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:50,938 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/b60c333832be4df3be7b4c33ca04ba84 is 50, key is test_row_0/B:col10/1733149188506/Put/seqid=0 2024-12-02T14:19:50,942 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202734ee8c615e74be1ba5e824b671237d8_367203f21af3d1a942fdfc82bb3c32fd store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:50,943 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202734ee8c615e74be1ba5e824b671237d8_367203f21af3d1a942fdfc82bb3c32fd, store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:50,944 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202734ee8c615e74be1ba5e824b671237d8_367203f21af3d1a942fdfc82bb3c32fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:50,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742047_1223 (size=13119) 2024-12-02T14:19:50,953 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/b60c333832be4df3be7b4c33ca04ba84 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/b60c333832be4df3be7b4c33ca04ba84 2024-12-02T14:19:50,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742048_1224 (size=4469) 2024-12-02T14:19:50,961 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#A#compaction#183 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:50,962 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/2f1913a7edb24e79aba505e5763b7a2b is 175, key is test_row_0/A:col10/1733149188506/Put/seqid=0 2024-12-02T14:19:50,964 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/B of 367203f21af3d1a942fdfc82bb3c32fd into b60c333832be4df3be7b4c33ca04ba84(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:50,964 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:50,964 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/B, priority=13, startTime=1733149190918; duration=0sec 2024-12-02T14:19:50,964 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:50,964 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:B 2024-12-02T14:19:50,964 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:50,965 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:50,965 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/C is initiating minor compaction (all files) 2024-12-02T14:19:50,965 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/C in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:50,965 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/bfbffa36591b4bfcb5bd3c02b05bce9f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/8bc24a4c86d44a31a36e2ac24fa04532, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4c21eb513ce6457fbbee92c06c3d4c0d] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=36.7 K 2024-12-02T14:19:50,966 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting bfbffa36591b4bfcb5bd3c02b05bce9f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733149185564 2024-12-02T14:19:50,966 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bc24a4c86d44a31a36e2ac24fa04532, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733149186299 2024-12-02T14:19:50,966 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c21eb513ce6457fbbee92c06c3d4c0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733149188502 2024-12-02T14:19:50,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742049_1225 (size=32073) 2024-12-02T14:19:50,973 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/2f1913a7edb24e79aba505e5763b7a2b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/2f1913a7edb24e79aba505e5763b7a2b 2024-12-02T14:19:50,976 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#C#compaction#185 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:50,976 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/98b016131deb4184ae5cc43e7cd32043 is 50, key is test_row_0/C:col10/1733149188506/Put/seqid=0 2024-12-02T14:19:50,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742050_1226 (size=13119) 2024-12-02T14:19:50,982 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/A of 367203f21af3d1a942fdfc82bb3c32fd into 2f1913a7edb24e79aba505e5763b7a2b(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:50,982 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:50,982 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/A, priority=13, startTime=1733149190918; duration=0sec 2024-12-02T14:19:50,982 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:50,982 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:A 2024-12-02T14:19:51,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T14:19:51,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:51,072 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T14:19:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:51,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025d6815a446a442b38b12934323d0b006_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149189686/Put/seqid=0 2024-12-02T14:19:51,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742051_1227 (size=12454) 2024-12-02T14:19:51,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:51,107 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025d6815a446a442b38b12934323d0b006_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025d6815a446a442b38b12934323d0b006_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:51,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/f8dbad65441744e6a5ed341b89b14ad1, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:51,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/f8dbad65441744e6a5ed341b89b14ad1 is 175, key is test_row_0/A:col10/1733149189686/Put/seqid=0 2024-12-02T14:19:51,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742052_1228 (size=31255) 2024-12-02T14:19:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:51,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. as already flushing 2024-12-02T14:19:51,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149251173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:51,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149251174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:51,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149251178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:51,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48354 deadline: 1733149251199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:51,203 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 367203f21af3d1a942fdfc82bb3c32fd of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18331 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T14:19:51,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149251289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:51,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149251289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:51,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149251289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:51,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:51,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:51,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:51,387 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/98b016131deb4184ae5cc43e7cd32043 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/98b016131deb4184ae5cc43e7cd32043 2024-12-02T14:19:51,405 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/C of 367203f21af3d1a942fdfc82bb3c32fd into 98b016131deb4184ae5cc43e7cd32043(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:51,405 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:51,405 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/C, priority=13, startTime=1733149190918; duration=0sec 2024-12-02T14:19:51,405 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:51,405 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:C 2024-12-02T14:19:51,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149251498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:51,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149251498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149251498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:51,524 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=389, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/f8dbad65441744e6a5ed341b89b14ad1 2024-12-02T14:19:51,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/131623cab9544feaaa2871374d88ea3f is 50, key is test_row_0/B:col10/1733149189686/Put/seqid=0 2024-12-02T14:19:51,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742053_1229 (size=12301) 2024-12-02T14:19:51,669 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-12-02T14:19:51,669 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-12-02T14:19:51,669 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:51,669 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:51,669 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:51,669 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:51,669 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:51,669 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:51,669 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:51,669 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:51,672 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-12-02T14:19:51,672 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:51,672 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:51,672 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:51,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:51,675 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-12-02T14:19:51,675 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:51,675 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:51,675 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:51,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T14:19:51,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:51,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:51,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149251808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149251808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149251808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:51,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48334 deadline: 1733149251848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:51,849 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 367203f21af3d1a942fdfc82bb3c32fd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:19:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:51,944 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/131623cab9544feaaa2871374d88ea3f 2024-12-02T14:19:51,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/5d4c3962e9b54b148525ea531f949720 is 50, key is test_row_0/C:col10/1733149189686/Put/seqid=0 2024-12-02T14:19:51,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742054_1230 (size=12301) 2024-12-02T14:19:52,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:52,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:52,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:19:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48336 deadline: 1733149252318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48370 deadline: 1733149252318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48430 deadline: 1733149252318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:19:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:19:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:19:52,358 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/5d4c3962e9b54b148525ea531f949720 2024-12-02T14:19:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/f8dbad65441744e6a5ed341b89b14ad1 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/f8dbad65441744e6a5ed341b89b14ad1 2024-12-02T14:19:52,366 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/f8dbad65441744e6a5ed341b89b14ad1, entries=150, sequenceid=389, filesize=30.5 K 2024-12-02T14:19:52,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/131623cab9544feaaa2871374d88ea3f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/131623cab9544feaaa2871374d88ea3f 2024-12-02T14:19:52,370 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/131623cab9544feaaa2871374d88ea3f, entries=150, sequenceid=389, filesize=12.0 K 2024-12-02T14:19:52,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/5d4c3962e9b54b148525ea531f949720 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/5d4c3962e9b54b148525ea531f949720 2024-12-02T14:19:52,374 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/5d4c3962e9b54b148525ea531f949720, entries=150, sequenceid=389, filesize=12.0 K 2024-12-02T14:19:52,374 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 367203f21af3d1a942fdfc82bb3c32fd in 1303ms, sequenceid=389, compaction requested=false 2024-12-02T14:19:52,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:52,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:52,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-02T14:19:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-02T14:19:52,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-02T14:19:52,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7650 sec 2024-12-02T14:19:52,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.7690 sec 2024-12-02T14:19:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T14:19:52,759 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:19:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:53,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T14:19:53,331 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-12-02T14:19:53,331 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-12-02T14:19:53,331 INFO [Thread-643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-643. 2024-12-02T14:19:53,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:19:53,331 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:53,331 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:53,331 DEBUG [Thread-643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:53,331 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:53,331 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:53,331 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:53,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:53,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:19:53,331 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:53,331 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:53,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:53,331 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:53,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:19:53,331 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:53,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:19:53,331 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:53,331 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:53,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b50bcb15cb5643a997f9f59334b6a530_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_0/A:col10/1733149191172/Put/seqid=0 2024-12-02T14:19:53,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742055_1231 (size=12454) 2024-12-02T14:19:53,744 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:19:53,748 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b50bcb15cb5643a997f9f59334b6a530_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b50bcb15cb5643a997f9f59334b6a530_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:19:53,749 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/064baac7b5db485d87423cc73097c7c9, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:53,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/064baac7b5db485d87423cc73097c7c9 is 175, key is test_row_0/A:col10/1733149191172/Put/seqid=0 2024-12-02T14:19:53,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742056_1232 (size=31255) 2024-12-02T14:19:53,870 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-12-02T14:19:53,870 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:19:53,870 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:53,870 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:19:53,870 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:19:54,153 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/064baac7b5db485d87423cc73097c7c9 2024-12-02T14:19:54,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/32e177370e1e4f9cac40d750506e4958 is 50, key is test_row_0/B:col10/1733149191172/Put/seqid=0 2024-12-02T14:19:54,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742057_1233 (size=12301) 2024-12-02T14:19:54,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/32e177370e1e4f9cac40d750506e4958 2024-12-02T14:19:54,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/61f4a69dab1248779e90f69f022f18aa is 50, key is test_row_0/C:col10/1733149191172/Put/seqid=0 2024-12-02T14:19:54,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742058_1234 (size=12301) 2024-12-02T14:19:54,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/61f4a69dab1248779e90f69f022f18aa 2024-12-02T14:19:54,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/064baac7b5db485d87423cc73097c7c9 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/064baac7b5db485d87423cc73097c7c9 2024-12-02T14:19:54,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/064baac7b5db485d87423cc73097c7c9, entries=150, sequenceid=410, filesize=30.5 K 2024-12-02T14:19:54,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/32e177370e1e4f9cac40d750506e4958 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/32e177370e1e4f9cac40d750506e4958 2024-12-02T14:19:55,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/32e177370e1e4f9cac40d750506e4958, entries=150, sequenceid=410, filesize=12.0 K 2024-12-02T14:19:55,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/61f4a69dab1248779e90f69f022f18aa as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/61f4a69dab1248779e90f69f022f18aa 2024-12-02T14:19:55,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/61f4a69dab1248779e90f69f022f18aa, entries=150, sequenceid=410, filesize=12.0 K 2024-12-02T14:19:55,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=6.71 KB/6870 for 367203f21af3d1a942fdfc82bb3c32fd in 1679ms, sequenceid=410, compaction requested=true 2024-12-02T14:19:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:19:55,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:55,010 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:55,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:19:55,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:55,010 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:55,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 367203f21af3d1a942fdfc82bb3c32fd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:19:55,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:55,011 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:55,011 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/B is initiating minor compaction (all files) 2024-12-02T14:19:55,011 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:55,011 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/B in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:55,011 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/A is initiating minor compaction (all files) 2024-12-02T14:19:55,011 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/A in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:55,011 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/b60c333832be4df3be7b4c33ca04ba84, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/131623cab9544feaaa2871374d88ea3f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/32e177370e1e4f9cac40d750506e4958] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=36.8 K 2024-12-02T14:19:55,011 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/2f1913a7edb24e79aba505e5763b7a2b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/f8dbad65441744e6a5ed341b89b14ad1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/064baac7b5db485d87423cc73097c7c9] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=92.4 K 2024-12-02T14:19:55,011 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:55,011 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/2f1913a7edb24e79aba505e5763b7a2b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/f8dbad65441744e6a5ed341b89b14ad1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/064baac7b5db485d87423cc73097c7c9] 2024-12-02T14:19:55,011 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting b60c333832be4df3be7b4c33ca04ba84, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733149188502 2024-12-02T14:19:55,011 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f1913a7edb24e79aba505e5763b7a2b, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733149188502 2024-12-02T14:19:55,011 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 131623cab9544feaaa2871374d88ea3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733149189677 2024-12-02T14:19:55,012 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8dbad65441744e6a5ed341b89b14ad1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733149189677 2024-12-02T14:19:55,012 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 32e177370e1e4f9cac40d750506e4958, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733149191172 2024-12-02T14:19:55,012 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 064baac7b5db485d87423cc73097c7c9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733149191172 2024-12-02T14:19:55,019 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#B#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:55,020 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/ac20e9b1e3204fe98f40a5d9e8bfa5aa is 50, key is test_row_0/B:col10/1733149191172/Put/seqid=0 2024-12-02T14:19:55,023 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:55,031 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202aa79b8c1041d439fa2e51a8bf5abbf2b_367203f21af3d1a942fdfc82bb3c32fd store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:55,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742059_1235 (size=13221) 2024-12-02T14:19:55,058 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202aa79b8c1041d439fa2e51a8bf5abbf2b_367203f21af3d1a942fdfc82bb3c32fd, store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:55,058 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202aa79b8c1041d439fa2e51a8bf5abbf2b_367203f21af3d1a942fdfc82bb3c32fd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:19:55,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742060_1236 (size=4469) 2024-12-02T14:19:55,440 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/ac20e9b1e3204fe98f40a5d9e8bfa5aa as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/ac20e9b1e3204fe98f40a5d9e8bfa5aa 2024-12-02T14:19:55,445 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/B of 367203f21af3d1a942fdfc82bb3c32fd into ac20e9b1e3204fe98f40a5d9e8bfa5aa(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:55,445 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:55,446 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/B, priority=13, startTime=1733149195010; duration=0sec 2024-12-02T14:19:55,446 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:19:55,446 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:B 2024-12-02T14:19:55,446 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:19:55,447 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:19:55,447 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 367203f21af3d1a942fdfc82bb3c32fd/C is initiating minor compaction (all files) 2024-12-02T14:19:55,447 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 367203f21af3d1a942fdfc82bb3c32fd/C in TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:19:55,447 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/98b016131deb4184ae5cc43e7cd32043, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/5d4c3962e9b54b148525ea531f949720, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/61f4a69dab1248779e90f69f022f18aa] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp, totalSize=36.8 K 2024-12-02T14:19:55,447 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 98b016131deb4184ae5cc43e7cd32043, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733149188502 2024-12-02T14:19:55,448 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d4c3962e9b54b148525ea531f949720, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733149189677 2024-12-02T14:19:55,448 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 61f4a69dab1248779e90f69f022f18aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733149191172 2024-12-02T14:19:55,456 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#C#compaction#194 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:55,456 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/244a3374760c4f93b832b98849a21045 is 50, key is test_row_0/C:col10/1733149191172/Put/seqid=0 2024-12-02T14:19:55,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742061_1237 (size=13221) 2024-12-02T14:19:55,463 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 367203f21af3d1a942fdfc82bb3c32fd#A#compaction#193 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:19:55,464 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/b1a85951ee254861a6f742ecd1bb01ad is 175, key is test_row_0/A:col10/1733149191172/Put/seqid=0 2024-12-02T14:19:55,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742062_1238 (size=32175) 2024-12-02T14:19:55,870 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/244a3374760c4f93b832b98849a21045 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/244a3374760c4f93b832b98849a21045 2024-12-02T14:19:55,876 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/b1a85951ee254861a6f742ecd1bb01ad as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/b1a85951ee254861a6f742ecd1bb01ad 2024-12-02T14:19:55,878 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/C of 367203f21af3d1a942fdfc82bb3c32fd into 244a3374760c4f93b832b98849a21045(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:55,878 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:55,878 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/C, priority=13, startTime=1733149195010; duration=0sec 2024-12-02T14:19:55,878 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:55,878 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:C 2024-12-02T14:19:55,883 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 367203f21af3d1a942fdfc82bb3c32fd/A of 367203f21af3d1a942fdfc82bb3c32fd into b1a85951ee254861a6f742ecd1bb01ad(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:19:55,883 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 367203f21af3d1a942fdfc82bb3c32fd: 2024-12-02T14:19:55,883 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., storeName=367203f21af3d1a942fdfc82bb3c32fd/A, priority=13, startTime=1733149195009; duration=0sec 2024-12-02T14:19:55,883 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:19:55,883 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 367203f21af3d1a942fdfc82bb3c32fd:A 2024-12-02T14:19:57,047 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T14:20:01,304 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-12-02T14:20:01,304 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:01,304 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:01,304 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:01,304 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:01,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T14:20:01,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-12-02T14:20:01,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-12-02T14:20:01,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 241 2024-12-02T14:20:01,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-12-02T14:20:01,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-02T14:20:01,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T14:20:01,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3700 2024-12-02T14:20:01,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3685 2024-12-02T14:20:01,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T14:20:01,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1662 2024-12-02T14:20:01,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4986 rows 2024-12-02T14:20:01,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1668 2024-12-02T14:20:01,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5004 rows 2024-12-02T14:20:01,305 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T14:20:01,306 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T14:20:01,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:01,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:01,306 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:01,307 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T14:20:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T14:20:01,311 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149201311"}]},"ts":"1733149201311"} 2024-12-02T14:20:01,313 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T14:20:01,313 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T14:20:01,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T14:20:01,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=367203f21af3d1a942fdfc82bb3c32fd, UNASSIGN}] 2024-12-02T14:20:01,317 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=367203f21af3d1a942fdfc82bb3c32fd, UNASSIGN 2024-12-02T14:20:01,318 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=367203f21af3d1a942fdfc82bb3c32fd, regionState=CLOSING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:01,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=367203f21af3d1a942fdfc82bb3c32fd, UNASSIGN because future has completed 2024-12-02T14:20:01,320 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T14:20:01,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:20:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T14:20:01,472 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:01,472 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T14:20:01,473 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 367203f21af3d1a942fdfc82bb3c32fd, disabling compactions & flushes 2024-12-02T14:20:01,473 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:20:01,473 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:20:01,473 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. after waiting 0 ms 2024-12-02T14:20:01,473 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:20:01,473 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 367203f21af3d1a942fdfc82bb3c32fd 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-02T14:20:01,473 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=A 2024-12-02T14:20:01,473 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:01,473 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=B 2024-12-02T14:20:01,473 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:01,473 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 367203f21af3d1a942fdfc82bb3c32fd, store=C 2024-12-02T14:20:01,473 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:01,479 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020f1c969cdde34e1c9dc983df480d7fe4_367203f21af3d1a942fdfc82bb3c32fd is 50, key is test_row_2/A:col10/1733149201300/Put/seqid=0 2024-12-02T14:20:01,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742063_1239 (size=7374) 2024-12-02T14:20:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T14:20:01,883 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:01,888 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020f1c969cdde34e1c9dc983df480d7fe4_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020f1c969cdde34e1c9dc983df480d7fe4_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:01,889 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/f41dfaec629b43dc887c23bc69a2f1ab, store: [table=TestAcidGuarantees family=A region=367203f21af3d1a942fdfc82bb3c32fd] 2024-12-02T14:20:01,890 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/f41dfaec629b43dc887c23bc69a2f1ab is 175, key is test_row_2/A:col10/1733149201300/Put/seqid=0 2024-12-02T14:20:01,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742064_1240 (size=13865) 2024-12-02T14:20:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T14:20:02,296 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=418, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/f41dfaec629b43dc887c23bc69a2f1ab 2024-12-02T14:20:02,309 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/eb7be9bf893a42748252e3de94f9315c is 50, key is test_row_2/B:col10/1733149201300/Put/seqid=0 2024-12-02T14:20:02,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742065_1241 (size=7415) 2024-12-02T14:20:02,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T14:20:02,714 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/eb7be9bf893a42748252e3de94f9315c 2024-12-02T14:20:02,728 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/13fc54b23e0844209a9e1e702b52d058 is 50, key is test_row_2/C:col10/1733149201300/Put/seqid=0 2024-12-02T14:20:02,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742066_1242 (size=7415) 2024-12-02T14:20:03,132 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/13fc54b23e0844209a9e1e702b52d058 2024-12-02T14:20:03,137 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/A/f41dfaec629b43dc887c23bc69a2f1ab as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/f41dfaec629b43dc887c23bc69a2f1ab 2024-12-02T14:20:03,142 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/f41dfaec629b43dc887c23bc69a2f1ab, entries=50, sequenceid=418, filesize=13.5 K 2024-12-02T14:20:03,143 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/B/eb7be9bf893a42748252e3de94f9315c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/eb7be9bf893a42748252e3de94f9315c 2024-12-02T14:20:03,148 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/eb7be9bf893a42748252e3de94f9315c, entries=50, sequenceid=418, filesize=7.2 K 2024-12-02T14:20:03,149 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/.tmp/C/13fc54b23e0844209a9e1e702b52d058 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/13fc54b23e0844209a9e1e702b52d058 2024-12-02T14:20:03,153 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/13fc54b23e0844209a9e1e702b52d058, entries=50, sequenceid=418, filesize=7.2 K 2024-12-02T14:20:03,153 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 367203f21af3d1a942fdfc82bb3c32fd in 1680ms, sequenceid=418, compaction requested=false 2024-12-02T14:20:03,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/18c40e20d4f6433e85ed1ea64000a427, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/b0cbd176fa874ec0b604ce6fab358d43, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/d8aa232c5064464ba964eb849a3403ed, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/1900f7f47e704698bfca5dafd22c22f5, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/0d20c8d11e4d4ad8b7eb1c5494613f01, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/3b57eccb8f8849e3a52833e4b73798b1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/6648dfba666e46fa828aa486c3f282c7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/47fb727654a04f37ad10d8a44ee5f91c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/dd7a9e0d93c64329ae1828789316c516, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/d4a89fbc574f459c92bfe6c58348fcb8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/e0e0779e1c40497581330f75b79ffbf8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/027cbdb6a1d0475d92b4675cc4039b84, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/e719f0679c3c4122804ff5fedf9e3f93, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/295e5bd0dde7413b9114eb1836a795a0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/9fd30e4bb79f4917a011599564d376c4, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/4fc85bfe01c542229b1d1566da72fa89, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/3ae87b65ef61443d9539c27592b01a65, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/53130dfd354a4427b25292ee5c613791, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/6557d1bb3a6d412eb6314560d8496392, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/0b6d8b06d2b4414b8ce632719b3a5b99, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/5e19e6c9c9954c068056ddc06fdb89b0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/9102eec7b24248deb1902bdf1d35d874, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/77d2f651334342f3be811733747dc29b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/33c7a257e7064573845d10ee966f55b1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/2f1913a7edb24e79aba505e5763b7a2b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/f8dbad65441744e6a5ed341b89b14ad1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/064baac7b5db485d87423cc73097c7c9] to archive 2024-12-02T14:20:03,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:20:03,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/18c40e20d4f6433e85ed1ea64000a427 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/18c40e20d4f6433e85ed1ea64000a427 2024-12-02T14:20:03,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/b0cbd176fa874ec0b604ce6fab358d43 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/b0cbd176fa874ec0b604ce6fab358d43 2024-12-02T14:20:03,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/d8aa232c5064464ba964eb849a3403ed to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/d8aa232c5064464ba964eb849a3403ed 2024-12-02T14:20:03,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/1900f7f47e704698bfca5dafd22c22f5 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/1900f7f47e704698bfca5dafd22c22f5 2024-12-02T14:20:03,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/0d20c8d11e4d4ad8b7eb1c5494613f01 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/0d20c8d11e4d4ad8b7eb1c5494613f01 2024-12-02T14:20:03,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/3b57eccb8f8849e3a52833e4b73798b1 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/3b57eccb8f8849e3a52833e4b73798b1 2024-12-02T14:20:03,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/6648dfba666e46fa828aa486c3f282c7 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/6648dfba666e46fa828aa486c3f282c7 2024-12-02T14:20:03,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/47fb727654a04f37ad10d8a44ee5f91c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/47fb727654a04f37ad10d8a44ee5f91c 2024-12-02T14:20:03,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/dd7a9e0d93c64329ae1828789316c516 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/dd7a9e0d93c64329ae1828789316c516 2024-12-02T14:20:03,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/d4a89fbc574f459c92bfe6c58348fcb8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/d4a89fbc574f459c92bfe6c58348fcb8 2024-12-02T14:20:03,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/e0e0779e1c40497581330f75b79ffbf8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/e0e0779e1c40497581330f75b79ffbf8 2024-12-02T14:20:03,176 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/027cbdb6a1d0475d92b4675cc4039b84 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/027cbdb6a1d0475d92b4675cc4039b84 2024-12-02T14:20:03,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/e719f0679c3c4122804ff5fedf9e3f93 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/e719f0679c3c4122804ff5fedf9e3f93 2024-12-02T14:20:03,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/295e5bd0dde7413b9114eb1836a795a0 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/295e5bd0dde7413b9114eb1836a795a0 2024-12-02T14:20:03,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/9fd30e4bb79f4917a011599564d376c4 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/9fd30e4bb79f4917a011599564d376c4 2024-12-02T14:20:03,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/4fc85bfe01c542229b1d1566da72fa89 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/4fc85bfe01c542229b1d1566da72fa89 2024-12-02T14:20:03,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/3ae87b65ef61443d9539c27592b01a65 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/3ae87b65ef61443d9539c27592b01a65 2024-12-02T14:20:03,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/53130dfd354a4427b25292ee5c613791 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/53130dfd354a4427b25292ee5c613791 2024-12-02T14:20:03,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/6557d1bb3a6d412eb6314560d8496392 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/6557d1bb3a6d412eb6314560d8496392 2024-12-02T14:20:03,185 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/0b6d8b06d2b4414b8ce632719b3a5b99 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/0b6d8b06d2b4414b8ce632719b3a5b99 2024-12-02T14:20:03,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/5e19e6c9c9954c068056ddc06fdb89b0 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/5e19e6c9c9954c068056ddc06fdb89b0 2024-12-02T14:20:03,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/9102eec7b24248deb1902bdf1d35d874 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/9102eec7b24248deb1902bdf1d35d874 2024-12-02T14:20:03,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/77d2f651334342f3be811733747dc29b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/77d2f651334342f3be811733747dc29b 2024-12-02T14:20:03,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/33c7a257e7064573845d10ee966f55b1 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/33c7a257e7064573845d10ee966f55b1 2024-12-02T14:20:03,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/2f1913a7edb24e79aba505e5763b7a2b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/2f1913a7edb24e79aba505e5763b7a2b 2024-12-02T14:20:03,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/f8dbad65441744e6a5ed341b89b14ad1 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/f8dbad65441744e6a5ed341b89b14ad1 2024-12-02T14:20:03,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/064baac7b5db485d87423cc73097c7c9 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/064baac7b5db485d87423cc73097c7c9 2024-12-02T14:20:03,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/9db191b33da54b3da7e195a5f4e3110e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5794c44ad3ce42609955111f2c71eb29, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3d7b0fb8086346beb5726ffeac283d70, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/57e80fa23547478b83d445d14ecd2b4b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/cfb013639aa942c2984365259859316b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/6095981bdd6743ee9ab32acc3ae5d74d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/373c530fe2d141a0b851d6dca95492c6, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3560ad536c5d4fbab0cbd7ef3cbe940a, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5caef67c4c7e440696599b08c5348817, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/fc16a84bfc604ee78acc8a61c5f578f2, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5293c652a711404e8b783a3d1d3f328d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3d9fd29af0de4169a1c6ad1c03fad06a, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/95f89958a15a434db2fb9137d16fae4f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/1ce9430306244a71a04681c81897b2c4, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5d41b44339814cce938c7b7c67e57a2a, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/e7cf5724fc114f3b928bf1f29275610a, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/39eef14864a14dcb9dead80305215be3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/20a619bc0f1242609e3ba0199b476a87, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/219bd5b0b2c649a7a84f18a28054c223, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/05d78a8c437e4cd1b8af024c3450f010, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/cafdf05a0b154f82ba837f52f01fa1e7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/2bc3f68d49464064b73b0e627c4c27cf, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/18ddaf7cacb34b3fb83f12bece75a12c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/b60c333832be4df3be7b4c33ca04ba84, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/2f2d6c130e7547208ff5de143838a94f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/131623cab9544feaaa2871374d88ea3f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/32e177370e1e4f9cac40d750506e4958] to archive 2024-12-02T14:20:03,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:20:03,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/9db191b33da54b3da7e195a5f4e3110e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/9db191b33da54b3da7e195a5f4e3110e 2024-12-02T14:20:03,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5794c44ad3ce42609955111f2c71eb29 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5794c44ad3ce42609955111f2c71eb29 2024-12-02T14:20:03,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3d7b0fb8086346beb5726ffeac283d70 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3d7b0fb8086346beb5726ffeac283d70 2024-12-02T14:20:03,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/57e80fa23547478b83d445d14ecd2b4b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/57e80fa23547478b83d445d14ecd2b4b 2024-12-02T14:20:03,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/cfb013639aa942c2984365259859316b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/cfb013639aa942c2984365259859316b 2024-12-02T14:20:03,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/6095981bdd6743ee9ab32acc3ae5d74d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/6095981bdd6743ee9ab32acc3ae5d74d 2024-12-02T14:20:03,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/373c530fe2d141a0b851d6dca95492c6 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/373c530fe2d141a0b851d6dca95492c6 2024-12-02T14:20:03,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3560ad536c5d4fbab0cbd7ef3cbe940a to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3560ad536c5d4fbab0cbd7ef3cbe940a 2024-12-02T14:20:03,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5caef67c4c7e440696599b08c5348817 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5caef67c4c7e440696599b08c5348817 2024-12-02T14:20:03,206 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/fc16a84bfc604ee78acc8a61c5f578f2 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/fc16a84bfc604ee78acc8a61c5f578f2 2024-12-02T14:20:03,207 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5293c652a711404e8b783a3d1d3f328d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5293c652a711404e8b783a3d1d3f328d 2024-12-02T14:20:03,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3d9fd29af0de4169a1c6ad1c03fad06a to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/3d9fd29af0de4169a1c6ad1c03fad06a 2024-12-02T14:20:03,209 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/95f89958a15a434db2fb9137d16fae4f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/95f89958a15a434db2fb9137d16fae4f 2024-12-02T14:20:03,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/1ce9430306244a71a04681c81897b2c4 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/1ce9430306244a71a04681c81897b2c4 2024-12-02T14:20:03,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5d41b44339814cce938c7b7c67e57a2a to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/5d41b44339814cce938c7b7c67e57a2a 2024-12-02T14:20:03,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/e7cf5724fc114f3b928bf1f29275610a to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/e7cf5724fc114f3b928bf1f29275610a 2024-12-02T14:20:03,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/39eef14864a14dcb9dead80305215be3 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/39eef14864a14dcb9dead80305215be3 2024-12-02T14:20:03,214 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/20a619bc0f1242609e3ba0199b476a87 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/20a619bc0f1242609e3ba0199b476a87 2024-12-02T14:20:03,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/219bd5b0b2c649a7a84f18a28054c223 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/219bd5b0b2c649a7a84f18a28054c223 2024-12-02T14:20:03,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/05d78a8c437e4cd1b8af024c3450f010 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/05d78a8c437e4cd1b8af024c3450f010 2024-12-02T14:20:03,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/cafdf05a0b154f82ba837f52f01fa1e7 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/cafdf05a0b154f82ba837f52f01fa1e7 2024-12-02T14:20:03,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/2bc3f68d49464064b73b0e627c4c27cf to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/2bc3f68d49464064b73b0e627c4c27cf 2024-12-02T14:20:03,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/18ddaf7cacb34b3fb83f12bece75a12c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/18ddaf7cacb34b3fb83f12bece75a12c 2024-12-02T14:20:03,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/b60c333832be4df3be7b4c33ca04ba84 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/b60c333832be4df3be7b4c33ca04ba84 2024-12-02T14:20:03,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/2f2d6c130e7547208ff5de143838a94f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/2f2d6c130e7547208ff5de143838a94f 2024-12-02T14:20:03,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/131623cab9544feaaa2871374d88ea3f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/131623cab9544feaaa2871374d88ea3f 2024-12-02T14:20:03,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/32e177370e1e4f9cac40d750506e4958 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/32e177370e1e4f9cac40d750506e4958 2024-12-02T14:20:03,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4e7670f601284f028a0b2aec428a04d8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/9f8108063ca943d596af44827d3cad63, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/76b7e43c322048c889bf39f4435b175e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/d6c843450a264f69af121a15a9a7b764, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/b961a273a0c444dcbb5927f37a650178, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/546b86a76d5d44ae9a224048832c8e10, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/8f434b14a2884803a189b37626dade18, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/bf2e2500a2d84084ab0d754d669b7105, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/0fe722e35c3047f78e8b5ccf8c685300, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/f06052c5b8044e0397381b7703af2e28, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/2a38f3a379d54590976119b74308a91e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4db5124d3c964ae59d19251c9c4506be, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/049b6c5a3e7d458fab8d2f3c5f522efc, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/a27c749a598547fb8b95072a74e28c6e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/33d38bbd282e4746acdc3f3b25193c1e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/b38443a537514ed8a044009516f82143, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/c595e88703f84c8ca2e9baaf9e36285f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/9366c1f0d0e8448cb9b7b67770d67f00, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/bb71b353337948ae954e1b756e9ee2ee, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/e04b2251b90447bebd71e0805f605a5e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/bfbffa36591b4bfcb5bd3c02b05bce9f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4dd9dc20a0814401904d66a1ca49e7a4, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/8bc24a4c86d44a31a36e2ac24fa04532, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/98b016131deb4184ae5cc43e7cd32043, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4c21eb513ce6457fbbee92c06c3d4c0d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/5d4c3962e9b54b148525ea531f949720, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/61f4a69dab1248779e90f69f022f18aa] to archive 2024-12-02T14:20:03,224 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:20:03,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4e7670f601284f028a0b2aec428a04d8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4e7670f601284f028a0b2aec428a04d8 2024-12-02T14:20:03,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/9f8108063ca943d596af44827d3cad63 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/9f8108063ca943d596af44827d3cad63 2024-12-02T14:20:03,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/76b7e43c322048c889bf39f4435b175e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/76b7e43c322048c889bf39f4435b175e 2024-12-02T14:20:03,228 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/d6c843450a264f69af121a15a9a7b764 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/d6c843450a264f69af121a15a9a7b764 2024-12-02T14:20:03,229 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/b961a273a0c444dcbb5927f37a650178 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/b961a273a0c444dcbb5927f37a650178 2024-12-02T14:20:03,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/546b86a76d5d44ae9a224048832c8e10 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/546b86a76d5d44ae9a224048832c8e10 2024-12-02T14:20:03,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/8f434b14a2884803a189b37626dade18 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/8f434b14a2884803a189b37626dade18 2024-12-02T14:20:03,232 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/bf2e2500a2d84084ab0d754d669b7105 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/bf2e2500a2d84084ab0d754d669b7105 2024-12-02T14:20:03,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/0fe722e35c3047f78e8b5ccf8c685300 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/0fe722e35c3047f78e8b5ccf8c685300 2024-12-02T14:20:03,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/f06052c5b8044e0397381b7703af2e28 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/f06052c5b8044e0397381b7703af2e28 2024-12-02T14:20:03,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/2a38f3a379d54590976119b74308a91e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/2a38f3a379d54590976119b74308a91e 2024-12-02T14:20:03,236 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4db5124d3c964ae59d19251c9c4506be to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4db5124d3c964ae59d19251c9c4506be 2024-12-02T14:20:03,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/049b6c5a3e7d458fab8d2f3c5f522efc to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/049b6c5a3e7d458fab8d2f3c5f522efc 2024-12-02T14:20:03,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/a27c749a598547fb8b95072a74e28c6e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/a27c749a598547fb8b95072a74e28c6e 2024-12-02T14:20:03,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/33d38bbd282e4746acdc3f3b25193c1e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/33d38bbd282e4746acdc3f3b25193c1e 2024-12-02T14:20:03,240 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/b38443a537514ed8a044009516f82143 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/b38443a537514ed8a044009516f82143 2024-12-02T14:20:03,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/c595e88703f84c8ca2e9baaf9e36285f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/c595e88703f84c8ca2e9baaf9e36285f 2024-12-02T14:20:03,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/9366c1f0d0e8448cb9b7b67770d67f00 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/9366c1f0d0e8448cb9b7b67770d67f00 2024-12-02T14:20:03,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/bb71b353337948ae954e1b756e9ee2ee to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/bb71b353337948ae954e1b756e9ee2ee 2024-12-02T14:20:03,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/e04b2251b90447bebd71e0805f605a5e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/e04b2251b90447bebd71e0805f605a5e 2024-12-02T14:20:03,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/bfbffa36591b4bfcb5bd3c02b05bce9f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/bfbffa36591b4bfcb5bd3c02b05bce9f 2024-12-02T14:20:03,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4dd9dc20a0814401904d66a1ca49e7a4 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4dd9dc20a0814401904d66a1ca49e7a4 2024-12-02T14:20:03,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/8bc24a4c86d44a31a36e2ac24fa04532 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/8bc24a4c86d44a31a36e2ac24fa04532 2024-12-02T14:20:03,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/98b016131deb4184ae5cc43e7cd32043 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/98b016131deb4184ae5cc43e7cd32043 2024-12-02T14:20:03,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4c21eb513ce6457fbbee92c06c3d4c0d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/4c21eb513ce6457fbbee92c06c3d4c0d 2024-12-02T14:20:03,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/5d4c3962e9b54b148525ea531f949720 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/5d4c3962e9b54b148525ea531f949720 2024-12-02T14:20:03,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/61f4a69dab1248779e90f69f022f18aa to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/61f4a69dab1248779e90f69f022f18aa 2024-12-02T14:20:03,256 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/recovered.edits/421.seqid, newMaxSeqId=421, maxSeqId=4 2024-12-02T14:20:03,256 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. 2024-12-02T14:20:03,257 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 367203f21af3d1a942fdfc82bb3c32fd: Waiting for close lock at 1733149201472Running coprocessor pre-close hooks at 1733149201473 (+1 ms)Disabling compacts and flushes for region at 1733149201473Disabling writes for close at 1733149201473Obtaining lock to block concurrent updates at 1733149201473Preparing flush snapshotting stores in 367203f21af3d1a942fdfc82bb3c32fd at 1733149201473Finished memstore snapshotting TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733149201473Flushing stores of TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd. at 1733149201474 (+1 ms)Flushing 367203f21af3d1a942fdfc82bb3c32fd/A: creating writer at 1733149201474Flushing mob file 367203f21af3d1a942fdfc82bb3c32fd/A: appending metadata at 1733149201479 (+5 ms)Flushing mob file 367203f21af3d1a942fdfc82bb3c32fd/A: closing flushed file at 1733149201479Flushing 367203f21af3d1a942fdfc82bb3c32fd/A: appending metadata at 1733149201889 (+410 ms)Flushing 367203f21af3d1a942fdfc82bb3c32fd/A: closing flushed file at 1733149201890 (+1 ms)Flushing 367203f21af3d1a942fdfc82bb3c32fd/B: creating writer at 1733149202306 (+416 ms)Flushing 367203f21af3d1a942fdfc82bb3c32fd/B: appending metadata at 1733149202308 (+2 ms)Flushing 367203f21af3d1a942fdfc82bb3c32fd/B: closing flushed file at 1733149202309 (+1 ms)Flushing 367203f21af3d1a942fdfc82bb3c32fd/C: creating writer at 1733149202725 (+416 ms)Flushing 367203f21af3d1a942fdfc82bb3c32fd/C: appending metadata at 1733149202727 (+2 ms)Flushing 367203f21af3d1a942fdfc82bb3c32fd/C: closing flushed file at 1733149202727Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@302efef9: reopening flushed file at 1733149203136 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3944b4df: reopening flushed file at 1733149203143 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e2b4c41: reopening flushed file at 1733149203148 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 367203f21af3d1a942fdfc82bb3c32fd in 1680ms, sequenceid=418, compaction requested=false at 1733149203153 (+5 ms)Writing region close event to WAL at 1733149203252 (+99 ms)Running coprocessor post-close hooks at 1733149203256 (+4 ms)Closed at 1733149203256 2024-12-02T14:20:03,259 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,259 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=367203f21af3d1a942fdfc82bb3c32fd, regionState=CLOSED 2024-12-02T14:20:03,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:20:03,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-02T14:20:03,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 367203f21af3d1a942fdfc82bb3c32fd, server=a3a61c9ba14f,46793,1733149138543 in 1.9410 sec 2024-12-02T14:20:03,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-02T14:20:03,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=367203f21af3d1a942fdfc82bb3c32fd, UNASSIGN in 1.9460 sec 2024-12-02T14:20:03,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-02T14:20:03,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9500 sec 2024-12-02T14:20:03,267 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149203267"}]},"ts":"1733149203267"} 2024-12-02T14:20:03,268 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T14:20:03,268 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T14:20:03,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9620 sec 2024-12-02T14:20:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T14:20:03,459 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:03,459 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T14:20:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:03,461 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-02T14:20:03,461 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:03,464 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,466 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/recovered.edits] 2024-12-02T14:20:03,469 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/b1a85951ee254861a6f742ecd1bb01ad to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/b1a85951ee254861a6f742ecd1bb01ad 2024-12-02T14:20:03,470 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/f41dfaec629b43dc887c23bc69a2f1ab to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/A/f41dfaec629b43dc887c23bc69a2f1ab 2024-12-02T14:20:03,472 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/ac20e9b1e3204fe98f40a5d9e8bfa5aa to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/ac20e9b1e3204fe98f40a5d9e8bfa5aa 2024-12-02T14:20:03,473 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/eb7be9bf893a42748252e3de94f9315c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/B/eb7be9bf893a42748252e3de94f9315c 2024-12-02T14:20:03,475 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/13fc54b23e0844209a9e1e702b52d058 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/13fc54b23e0844209a9e1e702b52d058 2024-12-02T14:20:03,476 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/244a3374760c4f93b832b98849a21045 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/C/244a3374760c4f93b832b98849a21045 2024-12-02T14:20:03,478 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/recovered.edits/421.seqid to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd/recovered.edits/421.seqid 2024-12-02T14:20:03,479 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,479 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T14:20:03,480 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T14:20:03,481 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T14:20:03,485 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020f1c969cdde34e1c9dc983df480d7fe4_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020f1c969cdde34e1c9dc983df480d7fe4_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,486 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120219d920d6eb7c483fb8ca52289fb7d185_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120219d920d6eb7c483fb8ca52289fb7d185_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,488 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022ad9c13c891b45a98245b7b355098b85_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022ad9c13c891b45a98245b7b355098b85_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,489 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120231b6768510464063ab9eb48d3d240be5_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120231b6768510464063ab9eb48d3d240be5_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,489 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120251c2db7e2f894e6090d45f14446005c0_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120251c2db7e2f894e6090d45f14446005c0_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,490 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120254742a1b63e846d7b4be5e936b3773b5_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120254742a1b63e846d7b4be5e936b3773b5_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,491 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025d6815a446a442b38b12934323d0b006_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025d6815a446a442b38b12934323d0b006_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,492 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025fbab95f558b4313a234331a0c00f2d6_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025fbab95f558b4313a234331a0c00f2d6_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,493 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202783a56121474481e9e1514adddcf11ad_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202783a56121474481e9e1514adddcf11ad_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,494 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202855f7cbdac9c4b30a39144351640bf68_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202855f7cbdac9c4b30a39144351640bf68_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,495 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a6b0a8156cef43faa66bc9ef2843b328_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a6b0a8156cef43faa66bc9ef2843b328_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,496 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b50bcb15cb5643a997f9f59334b6a530_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b50bcb15cb5643a997f9f59334b6a530_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,497 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d6947e12cca846d389154949bfd9adb9_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d6947e12cca846d389154949bfd9adb9_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,498 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d9931f30af584c15aa92e06d355fe7be_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d9931f30af584c15aa92e06d355fe7be_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,499 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202de66df25216b4a6887fb2428dd8a2eb8_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202de66df25216b4a6887fb2428dd8a2eb8_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,500 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202deff486488594c16abc77b890e7799f7_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202deff486488594c16abc77b890e7799f7_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,501 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e3407f926ec2492bb23c44caf5af8ce1_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e3407f926ec2492bb23c44caf5af8ce1_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,501 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e352ecd6a87c4eb3969498b2e4c43a0a_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e352ecd6a87c4eb3969498b2e4c43a0a_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,502 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ec93a8639fa04cc78c69462b03776abc_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ec93a8639fa04cc78c69462b03776abc_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,503 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f56bad6d4dd84b0a94e2a9fda9636cb6_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f56bad6d4dd84b0a94e2a9fda9636cb6_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,504 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fddacaccc00f4723943c09d3adde6512_367203f21af3d1a942fdfc82bb3c32fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fddacaccc00f4723943c09d3adde6512_367203f21af3d1a942fdfc82bb3c32fd 2024-12-02T14:20:03,504 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T14:20:03,506 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:03,508 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T14:20:03,510 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T14:20:03,511 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:03,511 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T14:20:03,511 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733149203511"}]},"ts":"9223372036854775807"} 2024-12-02T14:20:03,513 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T14:20:03,513 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 367203f21af3d1a942fdfc82bb3c32fd, NAME => 'TestAcidGuarantees,,1733149168496.367203f21af3d1a942fdfc82bb3c32fd.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T14:20:03,513 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T14:20:03,513 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733149203513"}]},"ts":"9223372036854775807"} 2024-12-02T14:20:03,515 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T14:20:03,516 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:03,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-12-02T14:20:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-02T14:20:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T14:20:03,569 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:03,578 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247 (was 248), OpenFileDescriptor=451 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=378 (was 449), ProcessCount=11 (was 11), AvailableMemoryMB=6084 (was 6205) 2024-12-02T14:20:03,587 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=247, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=378, ProcessCount=11, AvailableMemoryMB=6084 2024-12-02T14:20:03,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T14:20:03,588 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T14:20:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:03,591 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T14:20:03,591 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:03,591 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-12-02T14:20:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-02T14:20:03,592 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T14:20:03,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742067_1243 (size=1061) 2024-12-02T14:20:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-02T14:20:03,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-02T14:20:04,009 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919 2024-12-02T14:20:04,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742068_1244 (size=53) 2024-12-02T14:20:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-02T14:20:04,420 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:20:04,420 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c7cbc32e95f966c652e56aa6f1b3f54b, disabling compactions & flushes 2024-12-02T14:20:04,420 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:04,420 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:04,420 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. after waiting 0 ms 2024-12-02T14:20:04,420 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:04,420 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:04,420 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c7cbc32e95f966c652e56aa6f1b3f54b: Waiting for close lock at 1733149204420Disabling compacts and flushes for region at 1733149204420Disabling writes for close at 1733149204420Writing region close event to WAL at 1733149204420Closed at 1733149204420 2024-12-02T14:20:04,424 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T14:20:04,424 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733149204424"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733149204424"}]},"ts":"1733149204424"} 2024-12-02T14:20:04,428 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T14:20:04,429 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T14:20:04,429 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149204429"}]},"ts":"1733149204429"} 2024-12-02T14:20:04,431 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T14:20:04,432 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7cbc32e95f966c652e56aa6f1b3f54b, ASSIGN}] 2024-12-02T14:20:04,433 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7cbc32e95f966c652e56aa6f1b3f54b, ASSIGN 2024-12-02T14:20:04,434 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7cbc32e95f966c652e56aa6f1b3f54b, ASSIGN; state=OFFLINE, location=a3a61c9ba14f,46793,1733149138543; forceNewPlan=false, retain=false 2024-12-02T14:20:04,585 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=c7cbc32e95f966c652e56aa6f1b3f54b, regionState=OPENING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:04,590 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7cbc32e95f966c652e56aa6f1b3f54b, ASSIGN because future has completed 2024-12-02T14:20:04,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:20:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-02T14:20:04,748 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:04,748 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} 2024-12-02T14:20:04,748 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:04,748 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:20:04,748 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:04,748 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:04,750 INFO [StoreOpener-c7cbc32e95f966c652e56aa6f1b3f54b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:04,751 INFO [StoreOpener-c7cbc32e95f966c652e56aa6f1b3f54b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:20:04,751 INFO [StoreOpener-c7cbc32e95f966c652e56aa6f1b3f54b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7cbc32e95f966c652e56aa6f1b3f54b columnFamilyName A 2024-12-02T14:20:04,751 DEBUG [StoreOpener-c7cbc32e95f966c652e56aa6f1b3f54b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:04,751 INFO [StoreOpener-c7cbc32e95f966c652e56aa6f1b3f54b-1 {}] regionserver.HStore(327): Store=c7cbc32e95f966c652e56aa6f1b3f54b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:20:04,751 INFO [StoreOpener-c7cbc32e95f966c652e56aa6f1b3f54b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:04,752 INFO [StoreOpener-c7cbc32e95f966c652e56aa6f1b3f54b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:20:04,752 INFO [StoreOpener-c7cbc32e95f966c652e56aa6f1b3f54b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7cbc32e95f966c652e56aa6f1b3f54b columnFamilyName B 2024-12-02T14:20:04,752 DEBUG [StoreOpener-c7cbc32e95f966c652e56aa6f1b3f54b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:04,753 INFO [StoreOpener-c7cbc32e95f966c652e56aa6f1b3f54b-1 {}] regionserver.HStore(327): Store=c7cbc32e95f966c652e56aa6f1b3f54b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:20:04,753 INFO [StoreOpener-c7cbc32e95f966c652e56aa6f1b3f54b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:04,754 INFO [StoreOpener-c7cbc32e95f966c652e56aa6f1b3f54b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:20:04,754 INFO [StoreOpener-c7cbc32e95f966c652e56aa6f1b3f54b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7cbc32e95f966c652e56aa6f1b3f54b columnFamilyName C 2024-12-02T14:20:04,754 DEBUG [StoreOpener-c7cbc32e95f966c652e56aa6f1b3f54b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:04,754 INFO [StoreOpener-c7cbc32e95f966c652e56aa6f1b3f54b-1 {}] regionserver.HStore(327): Store=c7cbc32e95f966c652e56aa6f1b3f54b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:20:04,754 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:04,754 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:04,755 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:04,755 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:04,756 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:04,756 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:04,757 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T14:20:04,758 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:04,760 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T14:20:04,760 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened c7cbc32e95f966c652e56aa6f1b3f54b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71097803, jitterRate=0.059439823031425476}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T14:20:04,760 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:04,761 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for c7cbc32e95f966c652e56aa6f1b3f54b: Running coprocessor pre-open hook at 1733149204749Writing region info on filesystem at 1733149204749Initializing all the Stores at 1733149204749Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149204749Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149204750 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149204750Cleaning up temporary data from old regions at 1733149204756 (+6 ms)Running coprocessor post-open hooks at 1733149204760 (+4 ms)Region opened successfully at 1733149204761 (+1 ms) 2024-12-02T14:20:04,761 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., pid=61, masterSystemTime=1733149204745 2024-12-02T14:20:04,763 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:04,763 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:04,763 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=c7cbc32e95f966c652e56aa6f1b3f54b, regionState=OPEN, openSeqNum=2, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:04,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:20:04,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-02T14:20:04,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 in 175 msec 2024-12-02T14:20:04,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-02T14:20:04,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7cbc32e95f966c652e56aa6f1b3f54b, ASSIGN in 336 msec 2024-12-02T14:20:04,769 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T14:20:04,769 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149204769"}]},"ts":"1733149204769"} 2024-12-02T14:20:04,771 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T14:20:04,771 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T14:20:04,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-12-02T14:20:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-02T14:20:05,740 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:05,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24ada2c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,742 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:05,743 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:05,745 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:05,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:05,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:05,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@90629ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:05,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:05,748 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:05,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4529dcd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,749 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:05,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:05,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:05,754 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T14:20:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is a3a61c9ba14f,35877,1733149137941 2024-12-02T14:20:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@398c773d 2024-12-02T14:20:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T14:20:05,757 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T14:20:05,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fbd4712, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,758 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:05,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:05,758 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:05,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:05,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:05,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13b2a2a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:05,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:05,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:05,760 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa2df8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6f1434, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:05,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:05,761 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:05,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:05,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:05,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d57cdcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:05,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:05,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:05,762 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:05,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@708d11ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@634f1deb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,763 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:05,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:05,764 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@398c677b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:05,765 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:05,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a53fe7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec2896e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:05,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:05,766 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e1e1c11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:05,767 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76aa6896, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5155197b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,768 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:05,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:05,769 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:05,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62631f6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:05,771 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@357adc70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2108806c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,772 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:05,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:05,773 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f863988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:05,774 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:05,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57e3dfe8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6935c50e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,776 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:05,776 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:05,777 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:05,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:05,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:05,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@256f7e00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:05,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:05,778 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:05,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176809dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@767db965, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:05,780 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:05,780 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd87d54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:05,781 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc0d658, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6493a71c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,782 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:05,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:05,783 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:05,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:05,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:05,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14155993, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:05,784 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:05,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aa571d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a1b1c7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:05,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:05,786 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@331f7bc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:05,787 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:05,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da40ff1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:05,788 DEBUG [Thread-1134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:05,788 DEBUG [Thread-1142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:05,788 DEBUG [Thread-1153 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:05,788 DEBUG [Thread-1147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:05,788 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:20:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:05,789 DEBUG [Thread-1149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:05,789 DEBUG [Thread-1136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:05,790 DEBUG [Thread-1151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-02T14:20:05,790 DEBUG [Thread-1140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:05,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:05,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:05,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-02T14:20:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:05,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:05,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:05,792 DEBUG [Thread-1145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:05,792 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:20:05,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:05,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:05,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:05,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:05,793 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:20:05,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:20:05,793 DEBUG [Thread-1138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:20:05,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:05,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:20:05,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:20:05,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:20:05,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:20:05,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:05,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:20:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:20:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:20:05,797 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:20:05,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:20:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:05,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T14:20:05,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:05,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:05,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:05,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:05,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:05,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:05,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149265826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:05,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/9efb5e8636da44848291c308c68e63de is 50, key is test_row_0/A:col10/1733149205804/Put/seqid=0 2024-12-02T14:20:05,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149265828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:05,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:05,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149265828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:05,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149265831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149265829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:05,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742069_1245 (size=12001) 2024-12-02T14:20:05,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/9efb5e8636da44848291c308c68e63de 2024-12-02T14:20:05,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/3c4f3b6b16cc449487d28bced1556f6c is 50, key is test_row_0/B:col10/1733149205804/Put/seqid=0 2024-12-02T14:20:05,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742070_1246 (size=12001) 2024-12-02T14:20:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-02T14:20:05,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149265939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149265939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149265939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:05,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:05,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149265939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149265939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:05,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-02T14:20:05,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:05,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:05,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:05,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:05,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:06,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-02T14:20:06,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:06,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:06,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:06,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:06,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-02T14:20:06,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:06,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149266148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149266148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:06,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149266148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:06,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149266148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149266148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:06,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-02T14:20:06,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:06,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:06,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:06,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:06,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:06,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/3c4f3b6b16cc449487d28bced1556f6c 2024-12-02T14:20:06,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/a3231e9cd78b43398b03349ed9795327 is 50, key is test_row_0/C:col10/1733149205804/Put/seqid=0 2024-12-02T14:20:06,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742071_1247 (size=12001) 2024-12-02T14:20:06,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-02T14:20:06,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:06,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:06,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:06,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:06,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-02T14:20:06,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149266459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149266459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:06,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149266459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:06,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149266459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149266459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:06,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-02T14:20:06,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:06,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:06,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:06,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:06,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:06,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-02T14:20:06,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:06,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:06,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:06,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:06,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:06,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/a3231e9cd78b43398b03349ed9795327 2024-12-02T14:20:06,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/9efb5e8636da44848291c308c68e63de as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/9efb5e8636da44848291c308c68e63de 2024-12-02T14:20:06,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/9efb5e8636da44848291c308c68e63de, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T14:20:06,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/3c4f3b6b16cc449487d28bced1556f6c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/3c4f3b6b16cc449487d28bced1556f6c 2024-12-02T14:20:06,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/3c4f3b6b16cc449487d28bced1556f6c, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T14:20:06,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/a3231e9cd78b43398b03349ed9795327 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/a3231e9cd78b43398b03349ed9795327 2024-12-02T14:20:06,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/a3231e9cd78b43398b03349ed9795327, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T14:20:06,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c7cbc32e95f966c652e56aa6f1b3f54b in 929ms, sequenceid=13, compaction requested=false 2024-12-02T14:20:06,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:06,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-02T14:20:06,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:06,865 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T14:20:06,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:06,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:06,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:06,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:06,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:06,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:06,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/820e96cd564f4a689a4f0ed6c3ea4d58 is 50, key is test_row_0/A:col10/1733149205829/Put/seqid=0 2024-12-02T14:20:06,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742072_1248 (size=12001) 2024-12-02T14:20:06,876 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/820e96cd564f4a689a4f0ed6c3ea4d58 2024-12-02T14:20:06,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/c71dac672e0940bcb5fc31c85f410b6f is 50, key is test_row_0/B:col10/1733149205829/Put/seqid=0 2024-12-02T14:20:06,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742073_1249 (size=12001) 2024-12-02T14:20:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-02T14:20:06,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:06,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149266972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:06,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149266973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:06,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149266975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:06,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:06,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149266977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:06,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149266977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149267078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149267088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149267089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149267089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149267089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149267288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,292 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/c71dac672e0940bcb5fc31c85f410b6f 2024-12-02T14:20:07,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/c64b06c0db964977a2eecb128bf69a2b is 50, key is test_row_0/C:col10/1733149205829/Put/seqid=0 2024-12-02T14:20:07,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149267299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149267299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149267299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149267299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742074_1250 (size=12001) 2024-12-02T14:20:07,308 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/c64b06c0db964977a2eecb128bf69a2b 2024-12-02T14:20:07,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/820e96cd564f4a689a4f0ed6c3ea4d58 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/820e96cd564f4a689a4f0ed6c3ea4d58 2024-12-02T14:20:07,316 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/820e96cd564f4a689a4f0ed6c3ea4d58, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T14:20:07,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/c71dac672e0940bcb5fc31c85f410b6f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/c71dac672e0940bcb5fc31c85f410b6f 2024-12-02T14:20:07,321 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/c71dac672e0940bcb5fc31c85f410b6f, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T14:20:07,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/c64b06c0db964977a2eecb128bf69a2b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/c64b06c0db964977a2eecb128bf69a2b 2024-12-02T14:20:07,326 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/c64b06c0db964977a2eecb128bf69a2b, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T14:20:07,327 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c7cbc32e95f966c652e56aa6f1b3f54b in 462ms, sequenceid=33, compaction requested=false 2024-12-02T14:20:07,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:07,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:07,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-02T14:20:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-02T14:20:07,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-02T14:20:07,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5350 sec 2024-12-02T14:20:07,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.5430 sec 2024-12-02T14:20:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:07,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T14:20:07,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:07,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:07,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:07,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:07,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:07,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:07,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/b074995bbf2648068172f71bd22849a4 is 50, key is test_row_0/A:col10/1733149206975/Put/seqid=0 2024-12-02T14:20:07,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742075_1251 (size=12001) 2024-12-02T14:20:07,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149267619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149267621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149267622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149267623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149267623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149267729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149267729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149267729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149267729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149267729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-02T14:20:07,939 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:07,939 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:20:07,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-02T14:20:07,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T14:20:07,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149267939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,941 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:20:07,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149267939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149267939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149267939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,942 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:20:07,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:20:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:07,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:07,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149267939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:08,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/b074995bbf2648068172f71bd22849a4 2024-12-02T14:20:08,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/518c73c821a04538b3ed55ba4b80d7e8 is 50, key is test_row_0/B:col10/1733149206975/Put/seqid=0 2024-12-02T14:20:08,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742076_1252 (size=12001) 2024-12-02T14:20:08,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/518c73c821a04538b3ed55ba4b80d7e8 2024-12-02T14:20:08,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/bbbd7619e0be4182bcb432d314f7bebe is 50, key is test_row_0/C:col10/1733149206975/Put/seqid=0 2024-12-02T14:20:08,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742077_1253 (size=12001) 2024-12-02T14:20:08,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T14:20:08,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T14:20:08,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:08,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:08,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:08,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:08,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:08,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T14:20:08,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:08,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:08,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:08,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:08,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:08,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149268249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149268249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149268249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:08,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149268249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:08,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149268249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:08,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T14:20:08,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T14:20:08,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:08,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:08,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:08,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:08,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:08,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/bbbd7619e0be4182bcb432d314f7bebe 2024-12-02T14:20:08,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/b074995bbf2648068172f71bd22849a4 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b074995bbf2648068172f71bd22849a4 2024-12-02T14:20:08,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b074995bbf2648068172f71bd22849a4, entries=150, sequenceid=46, filesize=11.7 K 2024-12-02T14:20:08,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/518c73c821a04538b3ed55ba4b80d7e8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/518c73c821a04538b3ed55ba4b80d7e8 2024-12-02T14:20:08,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/518c73c821a04538b3ed55ba4b80d7e8, entries=150, sequenceid=46, filesize=11.7 K 2024-12-02T14:20:08,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/bbbd7619e0be4182bcb432d314f7bebe as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/bbbd7619e0be4182bcb432d314f7bebe 2024-12-02T14:20:08,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/bbbd7619e0be4182bcb432d314f7bebe, entries=150, sequenceid=46, filesize=11.7 K 2024-12-02T14:20:08,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c7cbc32e95f966c652e56aa6f1b3f54b in 871ms, sequenceid=46, compaction requested=true 2024-12-02T14:20:08,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:08,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:20:08,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:08,472 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:08,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:20:08,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:08,472 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:08,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:20:08,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:08,473 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:08,473 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/A is initiating minor compaction (all files) 2024-12-02T14:20:08,473 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/A in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:08,474 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/9efb5e8636da44848291c308c68e63de, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/820e96cd564f4a689a4f0ed6c3ea4d58, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b074995bbf2648068172f71bd22849a4] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=35.2 K 2024-12-02T14:20:08,474 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:08,474 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/B is initiating minor compaction (all files) 2024-12-02T14:20:08,474 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/B in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:08,474 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9efb5e8636da44848291c308c68e63de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733149205801 2024-12-02T14:20:08,474 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/3c4f3b6b16cc449487d28bced1556f6c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/c71dac672e0940bcb5fc31c85f410b6f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/518c73c821a04538b3ed55ba4b80d7e8] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=35.2 K 2024-12-02T14:20:08,474 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 820e96cd564f4a689a4f0ed6c3ea4d58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733149205820 2024-12-02T14:20:08,474 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c4f3b6b16cc449487d28bced1556f6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733149205801 2024-12-02T14:20:08,475 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting b074995bbf2648068172f71bd22849a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733149206975 2024-12-02T14:20:08,475 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting c71dac672e0940bcb5fc31c85f410b6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733149205820 2024-12-02T14:20:08,475 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 518c73c821a04538b3ed55ba4b80d7e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733149206975 2024-12-02T14:20:08,485 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#B#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:08,485 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/a42b8e3bc9424827bd5e7cbf969ab9d6 is 50, key is test_row_0/B:col10/1733149206975/Put/seqid=0 2024-12-02T14:20:08,489 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#A#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:08,490 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/c38510a098bf4bcc8a4693185f7ab13c is 50, key is test_row_0/A:col10/1733149206975/Put/seqid=0 2024-12-02T14:20:08,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742078_1254 (size=12104) 2024-12-02T14:20:08,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742079_1255 (size=12104) 2024-12-02T14:20:08,512 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/a42b8e3bc9424827bd5e7cbf969ab9d6 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/a42b8e3bc9424827bd5e7cbf969ab9d6 2024-12-02T14:20:08,529 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/c38510a098bf4bcc8a4693185f7ab13c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/c38510a098bf4bcc8a4693185f7ab13c 2024-12-02T14:20:08,533 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/B of c7cbc32e95f966c652e56aa6f1b3f54b into a42b8e3bc9424827bd5e7cbf969ab9d6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:08,533 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:08,533 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/B, priority=13, startTime=1733149208472; duration=0sec 2024-12-02T14:20:08,533 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:08,533 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:B 2024-12-02T14:20:08,533 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:08,535 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:08,535 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/C is initiating minor compaction (all files) 2024-12-02T14:20:08,535 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/C in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:08,535 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/a3231e9cd78b43398b03349ed9795327, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/c64b06c0db964977a2eecb128bf69a2b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/bbbd7619e0be4182bcb432d314f7bebe] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=35.2 K 2024-12-02T14:20:08,536 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting a3231e9cd78b43398b03349ed9795327, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733149205801 2024-12-02T14:20:08,537 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting c64b06c0db964977a2eecb128bf69a2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733149205820 2024-12-02T14:20:08,538 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting bbbd7619e0be4182bcb432d314f7bebe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733149206975 2024-12-02T14:20:08,539 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/A of c7cbc32e95f966c652e56aa6f1b3f54b into c38510a098bf4bcc8a4693185f7ab13c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:08,539 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:08,539 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/A, priority=13, startTime=1733149208472; duration=0sec 2024-12-02T14:20:08,540 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:08,540 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:A 2024-12-02T14:20:08,552 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#C#compaction#209 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:08,553 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/aa63f39833d24ec4b09ab3f6e0521433 is 50, key is test_row_0/C:col10/1733149206975/Put/seqid=0 2024-12-02T14:20:08,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T14:20:08,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:08,554 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T14:20:08,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:08,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:08,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:08,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:08,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:08,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:08,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742080_1256 (size=12104) 2024-12-02T14:20:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T14:20:08,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/01d2a7e995c948c2ac8133cb4bc222b7 is 50, key is test_row_0/A:col10/1733149207616/Put/seqid=0 2024-12-02T14:20:08,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742081_1257 (size=12001) 2024-12-02T14:20:08,589 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/01d2a7e995c948c2ac8133cb4bc222b7 2024-12-02T14:20:08,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/44b6283560bf4e95a4bc81aaa1cfaeb5 is 50, key is test_row_0/B:col10/1733149207616/Put/seqid=0 2024-12-02T14:20:08,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742082_1258 (size=12001) 2024-12-02T14:20:08,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:08,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:08,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:08,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149268766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:08,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:08,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149268766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:08,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:08,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:08,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:08,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149268767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:08,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:08,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:08,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149268767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:08,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:08,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:08,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149268768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:08,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:08,777 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T14:20:08,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:08,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149268878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149268878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149268878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149268878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149268879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:08,971 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/aa63f39833d24ec4b09ab3f6e0521433 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/aa63f39833d24ec4b09ab3f6e0521433 2024-12-02T14:20:08,975 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/C of c7cbc32e95f966c652e56aa6f1b3f54b into aa63f39833d24ec4b09ab3f6e0521433(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:08,975 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:08,975 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/C, priority=13, startTime=1733149208472; duration=0sec 2024-12-02T14:20:08,975 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:08,975 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:C 2024-12-02T14:20:09,007 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/44b6283560bf4e95a4bc81aaa1cfaeb5 2024-12-02T14:20:09,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/e80c7a83f1f64d17a370e777c7979497 is 50, key is test_row_0/C:col10/1733149207616/Put/seqid=0 2024-12-02T14:20:09,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742083_1259 (size=12001) 2024-12-02T14:20:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T14:20:09,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149269088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:09,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149269088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:09,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149269088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:09,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149269089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:09,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149269089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:09,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149269398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:09,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149269398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:09,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149269398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:09,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149269399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:09,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149269399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:09,419 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/e80c7a83f1f64d17a370e777c7979497 2024-12-02T14:20:09,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/01d2a7e995c948c2ac8133cb4bc222b7 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/01d2a7e995c948c2ac8133cb4bc222b7 2024-12-02T14:20:09,426 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/01d2a7e995c948c2ac8133cb4bc222b7, entries=150, sequenceid=67, filesize=11.7 K 2024-12-02T14:20:09,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/44b6283560bf4e95a4bc81aaa1cfaeb5 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/44b6283560bf4e95a4bc81aaa1cfaeb5 2024-12-02T14:20:09,431 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/44b6283560bf4e95a4bc81aaa1cfaeb5, entries=150, sequenceid=67, filesize=11.7 K 2024-12-02T14:20:09,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/e80c7a83f1f64d17a370e777c7979497 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/e80c7a83f1f64d17a370e777c7979497 2024-12-02T14:20:09,435 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/e80c7a83f1f64d17a370e777c7979497, entries=150, sequenceid=67, filesize=11.7 K 2024-12-02T14:20:09,436 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c7cbc32e95f966c652e56aa6f1b3f54b in 882ms, sequenceid=67, compaction requested=false 2024-12-02T14:20:09,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:09,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:09,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-02T14:20:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-02T14:20:09,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-02T14:20:09,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4950 sec 2024-12-02T14:20:09,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.5010 sec 2024-12-02T14:20:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:09,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T14:20:09,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:09,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:09,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:09,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:09,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:09,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:09,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/076e19747e274b36a942fc8d901a659b is 50, key is test_row_0/A:col10/1733149209911/Put/seqid=0 2024-12-02T14:20:09,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:09,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149269924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:09,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:09,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149269925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:09,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:09,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149269926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:09,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:09,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149269926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:09,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:09,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149269927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:09,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:09,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742084_1260 (size=14341) 2024-12-02T14:20:10,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149270038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:10,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149270039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:10,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149270039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:10,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149270039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:10,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:10,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149270039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:10,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:10,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T14:20:10,089 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:10,089 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:20:10,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-02T14:20:10,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T14:20:10,091 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:20:10,092 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:20:10,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:20:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T14:20:10,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T14:20:10,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:10,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:10,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:10,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:10,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:10,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149270248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:10,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149270248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149270248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149270248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149270249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:10,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/076e19747e274b36a942fc8d901a659b 2024-12-02T14:20:10,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/5b645633dac94a8ca2d065ac990d40b2 is 50, key is test_row_0/B:col10/1733149209911/Put/seqid=0 2024-12-02T14:20:10,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742085_1261 (size=12001) 2024-12-02T14:20:10,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T14:20:10,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:10,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:10,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:10,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:10,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T14:20:10,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T14:20:10,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:10,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:10,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:10,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:10,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:10,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149270558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:10,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149270558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149270558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:10,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:10,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149270558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149270558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:10,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T14:20:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:10,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:10,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:10,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T14:20:10,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/5b645633dac94a8ca2d065ac990d40b2 2024-12-02T14:20:10,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/96e8711184954a1292ea3d21a1de1680 is 50, key is test_row_0/C:col10/1733149209911/Put/seqid=0 2024-12-02T14:20:10,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742086_1262 (size=12001) 2024-12-02T14:20:10,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/96e8711184954a1292ea3d21a1de1680 2024-12-02T14:20:10,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/076e19747e274b36a942fc8d901a659b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/076e19747e274b36a942fc8d901a659b 2024-12-02T14:20:10,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/076e19747e274b36a942fc8d901a659b, entries=200, sequenceid=85, filesize=14.0 K 2024-12-02T14:20:10,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/5b645633dac94a8ca2d065ac990d40b2 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5b645633dac94a8ca2d065ac990d40b2 2024-12-02T14:20:10,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5b645633dac94a8ca2d065ac990d40b2, entries=150, sequenceid=85, filesize=11.7 K 2024-12-02T14:20:10,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/96e8711184954a1292ea3d21a1de1680 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/96e8711184954a1292ea3d21a1de1680 2024-12-02T14:20:10,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/96e8711184954a1292ea3d21a1de1680, entries=150, sequenceid=85, filesize=11.7 K 2024-12-02T14:20:10,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c7cbc32e95f966c652e56aa6f1b3f54b in 873ms, sequenceid=85, compaction requested=true 2024-12-02T14:20:10,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:10,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:20:10,785 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:10,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:10,785 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:10,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:20:10,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:10,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:20:10,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:10,785 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:10,785 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:10,785 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/A is initiating minor compaction (all files) 2024-12-02T14:20:10,785 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/B is initiating minor compaction (all files) 2024-12-02T14:20:10,786 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/A in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:10,786 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/B in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:10,786 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/c38510a098bf4bcc8a4693185f7ab13c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/01d2a7e995c948c2ac8133cb4bc222b7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/076e19747e274b36a942fc8d901a659b] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=37.5 K 2024-12-02T14:20:10,786 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/a42b8e3bc9424827bd5e7cbf969ab9d6, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/44b6283560bf4e95a4bc81aaa1cfaeb5, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5b645633dac94a8ca2d065ac990d40b2] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=35.3 K 2024-12-02T14:20:10,786 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting c38510a098bf4bcc8a4693185f7ab13c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733149206975 2024-12-02T14:20:10,786 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting a42b8e3bc9424827bd5e7cbf969ab9d6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733149206975 2024-12-02T14:20:10,789 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01d2a7e995c948c2ac8133cb4bc222b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733149207616 2024-12-02T14:20:10,789 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 44b6283560bf4e95a4bc81aaa1cfaeb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733149207616 2024-12-02T14:20:10,790 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b645633dac94a8ca2d065ac990d40b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733149208766 2024-12-02T14:20:10,790 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 076e19747e274b36a942fc8d901a659b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733149208766 2024-12-02T14:20:10,800 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#B#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:10,801 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/e90306f41fca4abdae0e847fed1a97c2 is 50, key is test_row_0/B:col10/1733149209911/Put/seqid=0 2024-12-02T14:20:10,802 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#A#compaction#216 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:10,802 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/c36808c25602430785413c448579fb3f is 50, key is test_row_0/A:col10/1733149209911/Put/seqid=0 2024-12-02T14:20:10,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742088_1264 (size=12207) 2024-12-02T14:20:10,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742087_1263 (size=12207) 2024-12-02T14:20:10,812 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/c36808c25602430785413c448579fb3f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/c36808c25602430785413c448579fb3f 2024-12-02T14:20:10,817 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/A of c7cbc32e95f966c652e56aa6f1b3f54b into c36808c25602430785413c448579fb3f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:10,817 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:10,817 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/A, priority=13, startTime=1733149210784; duration=0sec 2024-12-02T14:20:10,817 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:10,817 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:A 2024-12-02T14:20:10,817 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:10,818 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:10,818 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/C is initiating minor compaction (all files) 2024-12-02T14:20:10,818 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/C in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:10,818 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/aa63f39833d24ec4b09ab3f6e0521433, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/e80c7a83f1f64d17a370e777c7979497, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/96e8711184954a1292ea3d21a1de1680] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=35.3 K 2024-12-02T14:20:10,819 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa63f39833d24ec4b09ab3f6e0521433, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733149206975 2024-12-02T14:20:10,819 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting e80c7a83f1f64d17a370e777c7979497, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733149207616 2024-12-02T14:20:10,819 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96e8711184954a1292ea3d21a1de1680, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733149208766 2024-12-02T14:20:10,826 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#C#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:10,827 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/0546bf93956b4fc985caf081a01094cf is 50, key is test_row_0/C:col10/1733149209911/Put/seqid=0 2024-12-02T14:20:10,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742089_1265 (size=12207) 2024-12-02T14:20:10,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T14:20:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:10,858 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T14:20:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:10,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/12881aa8919d44a787b077e8ebf3e10d is 50, key is test_row_0/A:col10/1733149209924/Put/seqid=0 2024-12-02T14:20:10,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742090_1266 (size=12001) 2024-12-02T14:20:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:11,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:11,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149271085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149271087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149271087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149271088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149271088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149271198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149271198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149271199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149271199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149271199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,211 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/e90306f41fca4abdae0e847fed1a97c2 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/e90306f41fca4abdae0e847fed1a97c2 2024-12-02T14:20:11,215 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/B of c7cbc32e95f966c652e56aa6f1b3f54b into e90306f41fca4abdae0e847fed1a97c2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:11,215 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:11,215 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/B, priority=13, startTime=1733149210785; duration=0sec 2024-12-02T14:20:11,215 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:11,215 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:B 2024-12-02T14:20:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T14:20:11,235 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/0546bf93956b4fc985caf081a01094cf as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/0546bf93956b4fc985caf081a01094cf 2024-12-02T14:20:11,239 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/C of c7cbc32e95f966c652e56aa6f1b3f54b into 0546bf93956b4fc985caf081a01094cf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:11,240 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:11,240 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/C, priority=13, startTime=1733149210785; duration=0sec 2024-12-02T14:20:11,240 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:11,240 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:C 2024-12-02T14:20:11,266 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/12881aa8919d44a787b077e8ebf3e10d 2024-12-02T14:20:11,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/584e980d43284412b9cdb2ef21d3c89c is 50, key is test_row_0/B:col10/1733149209924/Put/seqid=0 2024-12-02T14:20:11,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742091_1267 (size=12001) 2024-12-02T14:20:11,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149271408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149271408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149271409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149271409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149271409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,677 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/584e980d43284412b9cdb2ef21d3c89c 2024-12-02T14:20:11,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/d7761e85977443c199683254340e3e97 is 50, key is test_row_0/C:col10/1733149209924/Put/seqid=0 2024-12-02T14:20:11,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742092_1268 (size=12001) 2024-12-02T14:20:11,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149271719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149271719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149271719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149271719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:11,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149271720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:11,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:11,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:11,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,099 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/d7761e85977443c199683254340e3e97 2024-12-02T14:20:12,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/12881aa8919d44a787b077e8ebf3e10d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/12881aa8919d44a787b077e8ebf3e10d 2024-12-02T14:20:12,116 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/12881aa8919d44a787b077e8ebf3e10d, entries=150, sequenceid=101, filesize=11.7 K 2024-12-02T14:20:12,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/584e980d43284412b9cdb2ef21d3c89c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/584e980d43284412b9cdb2ef21d3c89c 2024-12-02T14:20:12,122 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/584e980d43284412b9cdb2ef21d3c89c, entries=150, sequenceid=101, filesize=11.7 K 2024-12-02T14:20:12,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/d7761e85977443c199683254340e3e97 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/d7761e85977443c199683254340e3e97 2024-12-02T14:20:12,129 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/d7761e85977443c199683254340e3e97, entries=150, sequenceid=101, filesize=11.7 K 2024-12-02T14:20:12,130 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for c7cbc32e95f966c652e56aa6f1b3f54b in 1272ms, sequenceid=101, compaction requested=false 2024-12-02T14:20:12,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:12,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:12,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-02T14:20:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-02T14:20:12,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-02T14:20:12,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0390 sec 2024-12-02T14:20:12,135 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.0450 sec 2024-12-02T14:20:12,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T14:20:12,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:12,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:12,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:12,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:12,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:12,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:12,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T14:20:12,239 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:12,239 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:20:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-02T14:20:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T14:20:12,244 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:20:12,245 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:20:12,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:20:12,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/b8d49a4cc7b2451aae5668c36c1709b3 is 50, key is test_row_0/A:col10/1733149212233/Put/seqid=0 2024-12-02T14:20:12,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149272251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149272254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149272263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149272267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149272275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742093_1269 (size=12001) 2024-12-02T14:20:12,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/b8d49a4cc7b2451aae5668c36c1709b3 2024-12-02T14:20:12,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/d57a454a8e714202a73451e0eeddfdca is 50, key is test_row_0/B:col10/1733149212233/Put/seqid=0 2024-12-02T14:20:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T14:20:12,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742094_1270 (size=12001) 2024-12-02T14:20:12,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149272369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149272369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149272379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149272379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149272389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T14:20:12,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:12,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:12,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:12,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:12,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:12,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T14:20:12,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:12,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:12,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:12,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:12,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T14:20:12,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149272579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149272581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149272588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149272588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149272598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T14:20:12,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:12,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:12,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:12,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:12,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:12,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/d57a454a8e714202a73451e0eeddfdca 2024-12-02T14:20:12,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/65a27c567ea143dfa21175f9f778a326 is 50, key is test_row_0/C:col10/1733149212233/Put/seqid=0 2024-12-02T14:20:12,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742095_1271 (size=12001) 2024-12-02T14:20:12,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T14:20:12,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:12,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:12,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:12,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:12,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T14:20:12,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149272889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149272889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149272898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149272898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:12,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:12,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149272908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:13,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T14:20:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:13,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:13,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:13,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T14:20:13,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:13,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:13,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:13,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:13,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:13,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:13,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/65a27c567ea143dfa21175f9f778a326 2024-12-02T14:20:13,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/b8d49a4cc7b2451aae5668c36c1709b3 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b8d49a4cc7b2451aae5668c36c1709b3 2024-12-02T14:20:13,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b8d49a4cc7b2451aae5668c36c1709b3, entries=150, sequenceid=123, filesize=11.7 K 2024-12-02T14:20:13,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/d57a454a8e714202a73451e0eeddfdca as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/d57a454a8e714202a73451e0eeddfdca 2024-12-02T14:20:13,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/d57a454a8e714202a73451e0eeddfdca, entries=150, sequenceid=123, filesize=11.7 K 2024-12-02T14:20:13,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/65a27c567ea143dfa21175f9f778a326 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/65a27c567ea143dfa21175f9f778a326 2024-12-02T14:20:13,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/65a27c567ea143dfa21175f9f778a326, entries=150, sequenceid=123, filesize=11.7 K 2024-12-02T14:20:13,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for c7cbc32e95f966c652e56aa6f1b3f54b in 1013ms, sequenceid=123, compaction requested=true 2024-12-02T14:20:13,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:13,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:20:13,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:13,246 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:13,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:20:13,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:13,246 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:13,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:20:13,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:13,248 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:13,248 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/B is initiating minor compaction (all files) 2024-12-02T14:20:13,248 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/B in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:13,249 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/e90306f41fca4abdae0e847fed1a97c2, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/584e980d43284412b9cdb2ef21d3c89c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/d57a454a8e714202a73451e0eeddfdca] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=35.4 K 2024-12-02T14:20:13,250 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:13,250 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/A is initiating minor compaction (all files) 2024-12-02T14:20:13,250 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/A in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:13,250 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/c36808c25602430785413c448579fb3f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/12881aa8919d44a787b077e8ebf3e10d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b8d49a4cc7b2451aae5668c36c1709b3] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=35.4 K 2024-12-02T14:20:13,250 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting e90306f41fca4abdae0e847fed1a97c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733149208766 2024-12-02T14:20:13,252 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting c36808c25602430785413c448579fb3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733149208766 2024-12-02T14:20:13,254 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 584e980d43284412b9cdb2ef21d3c89c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733149209924 2024-12-02T14:20:13,254 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12881aa8919d44a787b077e8ebf3e10d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733149209924 2024-12-02T14:20:13,254 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting d57a454a8e714202a73451e0eeddfdca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733149211071 2024-12-02T14:20:13,256 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8d49a4cc7b2451aae5668c36c1709b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733149211071 2024-12-02T14:20:13,265 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#B#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:13,266 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/2d27460d1c20424d8f92feb0c777212a is 50, key is test_row_0/B:col10/1733149212233/Put/seqid=0 2024-12-02T14:20:13,266 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#A#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:13,267 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/350ee6e94cd54e649fae61db2fdb83fa is 50, key is test_row_0/A:col10/1733149212233/Put/seqid=0 2024-12-02T14:20:13,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742097_1273 (size=12309) 2024-12-02T14:20:13,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742096_1272 (size=12309) 2024-12-02T14:20:13,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T14:20:13,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:13,326 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T14:20:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:13,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/55e62124507946f3ae39e5ff9199859f is 50, key is test_row_0/A:col10/1733149212249/Put/seqid=0 2024-12-02T14:20:13,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742098_1274 (size=12151) 2024-12-02T14:20:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T14:20:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:13,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:13,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149273432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:13,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149273445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:13,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149273445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:13,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149273447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:13,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149273447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:13,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149273558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:13,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:13,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149273559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149273559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149273559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:13,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149273559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:13,711 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/350ee6e94cd54e649fae61db2fdb83fa as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/350ee6e94cd54e649fae61db2fdb83fa 2024-12-02T14:20:13,711 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/2d27460d1c20424d8f92feb0c777212a as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/2d27460d1c20424d8f92feb0c777212a 2024-12-02T14:20:13,716 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/A of c7cbc32e95f966c652e56aa6f1b3f54b into 350ee6e94cd54e649fae61db2fdb83fa(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:13,716 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/B of c7cbc32e95f966c652e56aa6f1b3f54b into 2d27460d1c20424d8f92feb0c777212a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:13,716 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:13,716 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:13,716 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/A, priority=13, startTime=1733149213246; duration=0sec 2024-12-02T14:20:13,716 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/B, priority=13, startTime=1733149213246; duration=0sec 2024-12-02T14:20:13,716 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:13,716 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:A 2024-12-02T14:20:13,716 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:13,716 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:B 2024-12-02T14:20:13,716 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:13,717 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:13,717 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/C is initiating minor compaction (all files) 2024-12-02T14:20:13,717 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/C in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:13,717 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/0546bf93956b4fc985caf081a01094cf, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/d7761e85977443c199683254340e3e97, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/65a27c567ea143dfa21175f9f778a326] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=35.4 K 2024-12-02T14:20:13,718 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0546bf93956b4fc985caf081a01094cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733149208766 2024-12-02T14:20:13,718 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7761e85977443c199683254340e3e97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733149209924 2024-12-02T14:20:13,718 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65a27c567ea143dfa21175f9f778a326, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733149211071 2024-12-02T14:20:13,724 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#C#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:13,724 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/f944e0425d2a4498a9565686f18d89e3 is 50, key is test_row_0/C:col10/1733149212233/Put/seqid=0 2024-12-02T14:20:13,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742099_1275 (size=12309) 2024-12-02T14:20:13,767 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/55e62124507946f3ae39e5ff9199859f 2024-12-02T14:20:13,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:13,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149273768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:13,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149273769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149273769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:13,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149273769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149273769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:13,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/cc74ae0df85343c8a0674cbf30c94592 is 50, key is test_row_0/B:col10/1733149212249/Put/seqid=0 2024-12-02T14:20:13,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742100_1276 (size=12151) 2024-12-02T14:20:14,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149274078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:14,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:14,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149274078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:14,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149274078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149274079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149274079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:14,134 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/f944e0425d2a4498a9565686f18d89e3 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/f944e0425d2a4498a9565686f18d89e3 2024-12-02T14:20:14,139 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/C of c7cbc32e95f966c652e56aa6f1b3f54b into f944e0425d2a4498a9565686f18d89e3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:14,139 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:14,139 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/C, priority=13, startTime=1733149213246; duration=0sec 2024-12-02T14:20:14,139 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:14,139 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:C 2024-12-02T14:20:14,178 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/cc74ae0df85343c8a0674cbf30c94592 2024-12-02T14:20:14,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/39992cdc15c64d95a208a96454caa81b is 50, key is test_row_0/C:col10/1733149212249/Put/seqid=0 2024-12-02T14:20:14,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742101_1277 (size=12151) 2024-12-02T14:20:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T14:20:14,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149274588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:14,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149274588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:14,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:14,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149274588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149274588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149274588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:14,592 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/39992cdc15c64d95a208a96454caa81b 2024-12-02T14:20:14,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/55e62124507946f3ae39e5ff9199859f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/55e62124507946f3ae39e5ff9199859f 2024-12-02T14:20:14,600 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/55e62124507946f3ae39e5ff9199859f, entries=150, sequenceid=137, filesize=11.9 K 2024-12-02T14:20:14,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/cc74ae0df85343c8a0674cbf30c94592 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/cc74ae0df85343c8a0674cbf30c94592 2024-12-02T14:20:14,604 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/cc74ae0df85343c8a0674cbf30c94592, entries=150, sequenceid=137, filesize=11.9 K 2024-12-02T14:20:14,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/39992cdc15c64d95a208a96454caa81b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/39992cdc15c64d95a208a96454caa81b 2024-12-02T14:20:14,608 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/39992cdc15c64d95a208a96454caa81b, entries=150, sequenceid=137, filesize=11.9 K 2024-12-02T14:20:14,609 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for c7cbc32e95f966c652e56aa6f1b3f54b in 1283ms, sequenceid=137, compaction requested=false 2024-12-02T14:20:14,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:14,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:14,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-02T14:20:14,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-02T14:20:14,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-02T14:20:14,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3650 sec 2024-12-02T14:20:14,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.3730 sec 2024-12-02T14:20:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:15,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T14:20:15,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149213431, max timestamp=1733149215600Num uniques 859; 2024-12-02T14:20:15,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:20:15,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149213431, max timestamp=1733149215600Num uniques 859; 2024-12-02T14:20:15,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:20:15,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733149213431, max timestamp=1733149215600Num uniques 859; 2024-12-02T14:20:15,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T14:20:15,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:15,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:15,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:15,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:15,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:15,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:15,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/0bb79eee86554c2ba513eee36e45a12b is 50, key is test_row_0/A:col10/1733149213445/Put/seqid=0 2024-12-02T14:20:15,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742102_1278 (size=14541) 2024-12-02T14:20:15,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733149215606, max timestamp=1733149215628Num uniques 859; 2024-12-02T14:20:15,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733149215606, max timestamp=1733149215628Num uniques 859; 2024-12-02T14:20:15,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T14:20:15,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T14:20:15,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733149215606, max timestamp=1733149215628Num uniques 859; 2024-12-02T14:20:15,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T14:20:15,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733149215674, max timestamp=1733149215681Num uniques 1118; 2024-12-02T14:20:15,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733149215674, max timestamp=1733149215681Num uniques 1118; 2024-12-02T14:20:15,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-02T14:20:15,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-02T14:20:15,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733149215674, max timestamp=1733149215681Num uniques 1118; 2024-12-02T14:20:15,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-02T14:20:15,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149275703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:15,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149275704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:15,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733149215695, max timestamp=1733149215708Num uniques 1077; 2024-12-02T14:20:15,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-02T14:20:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:15,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733149215695, max timestamp=1733149215708Num uniques 1077; 2024-12-02T14:20:15,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-02T14:20:15,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733149215695, max timestamp=1733149215708Num uniques 1077; 2024-12-02T14:20:15,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-02T14:20:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:15,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733149215733, max timestamp=1733149215739Num uniques 1036; 2024-12-02T14:20:15,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-02T14:20:15,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733149215733, max timestamp=1733149215739Num uniques 1036; 2024-12-02T14:20:15,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-02T14:20:15,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733149215733, max timestamp=1733149215739Num uniques 1036; 2024-12-02T14:20:15,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-02T14:20:15,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733149215767, max timestamp=1733149215814Num uniques 1095; 2024-12-02T14:20:15,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-02T14:20:15,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733149215767, max timestamp=1733149215814Num uniques 1095; 2024-12-02T14:20:15,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-02T14:20:15,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733149215767, max timestamp=1733149215814Num uniques 1095; 2024-12-02T14:20:15,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-02T14:20:15,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149275840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:15,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149275840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:15,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149275842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:15,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733149215829, max timestamp=1733149215846Num uniques 1154; 2024-12-02T14:20:15,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-12-02T14:20:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:15,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733149215829, max timestamp=1733149215846Num uniques 1154; 2024-12-02T14:20:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:15,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-12-02T14:20:15,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733149215829, max timestamp=1733149215846Num uniques 1154; 2024-12-02T14:20:15,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-12-02T14:20:15,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7455, totalHeapSize=17192, min timestamp=1733149215870, max timestamp=1733149215898Num uniques 1063; 2024-12-02T14:20:15,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48675, new segment data size=7455, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-02T14:20:15,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7456, totalHeapSize=17192, min timestamp=1733149215870, max timestamp=1733149215898Num uniques 1063; 2024-12-02T14:20:15,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48676, new segment data size=7456, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-02T14:20:15,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733149215870, max timestamp=1733149215898Num uniques 1063; 2024-12-02T14:20:15,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-02T14:20:15,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733149215926, max timestamp=1733149215938Num uniques 1022; 2024-12-02T14:20:15,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-02T14:20:15,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733149215926, max timestamp=1733149215938Num uniques 1022; 2024-12-02T14:20:15,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-02T14:20:15,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733149215926, max timestamp=1733149215938Num uniques 1022; 2024-12-02T14:20:15,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-02T14:20:15,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733149215962, max timestamp=1733149215968Num uniques 1031; 2024-12-02T14:20:15,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-02T14:20:15,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733149215962, max timestamp=1733149215968Num uniques 1031; 2024-12-02T14:20:15,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-02T14:20:15,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733149215962, max timestamp=1733149215968Num uniques 1031; 2024-12-02T14:20:15,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-02T14:20:15,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733149215982, max timestamp=1733149215991Num uniques 1040; 2024-12-02T14:20:15,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-02T14:20:15,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733149215982, max timestamp=1733149215991Num uniques 1040; 2024-12-02T14:20:15,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-02T14:20:15,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733149215982, max timestamp=1733149215991Num uniques 1040; 2024-12-02T14:20:15,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-02T14:20:16,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733149216003, max timestamp=1733149216009Num uniques 1049; 2024-12-02T14:20:16,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-02T14:20:16,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733149216003, max timestamp=1733149216009Num uniques 1049; 2024-12-02T14:20:16,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-02T14:20:16,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733149216003, max timestamp=1733149216009Num uniques 1049; 2024-12-02T14:20:16,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-02T14:20:16,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/0bb79eee86554c2ba513eee36e45a12b 2024-12-02T14:20:16,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/11aa8388b4e94318bd8b712d0eb879ce is 50, key is test_row_0/B:col10/1733149213445/Put/seqid=0 2024-12-02T14:20:16,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742103_1279 (size=12151) 2024-12-02T14:20:16,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/11aa8388b4e94318bd8b712d0eb879ce 2024-12-02T14:20:16,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149276028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149276029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149276029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149276031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149276034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/6025cb8ac0e344f5b61a9efffdde67fd is 50, key is test_row_0/C:col10/1733149213445/Put/seqid=0 2024-12-02T14:20:16,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742104_1280 (size=12151) 2024-12-02T14:20:16,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733149216027, max timestamp=1733149216029Num uniques 1058; 2024-12-02T14:20:16,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=6870, suffix heap size=138000,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=150 2024-12-02T14:20:16,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733149216027, max timestamp=1733149216029Num uniques 1058; 2024-12-02T14:20:16,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=6870, suffix heap size=138000,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=150 2024-12-02T14:20:16,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733149216027, max timestamp=1733149216029Num uniques 1058; 2024-12-02T14:20:16,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=6870, suffix heap size=138000,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=150 2024-12-02T14:20:16,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149276153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149276154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733149216152, max timestamp=1733149216155Num uniques 1009; 2024-12-02T14:20:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149276155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46212, new segment data size=7642, suffix heap size=132904,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=167 2024-12-02T14:20:16,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733149216152, max timestamp=1733149216155Num uniques 1009; 2024-12-02T14:20:16,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733149216152, max timestamp=1733149216155Num uniques 1009; 2024-12-02T14:20:16,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46212, new segment data size=7642, suffix heap size=132904,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=167 2024-12-02T14:20:16,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46212, new segment data size=7642, suffix heap size=132904,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=167 2024-12-02T14:20:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733149216180, max timestamp=1733149216193Num uniques 1026; 2024-12-02T14:20:16,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-12-02T14:20:16,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733149216180, max timestamp=1733149216193Num uniques 1026; 2024-12-02T14:20:16,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-12-02T14:20:16,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733149216180, max timestamp=1733149216193Num uniques 1026; 2024-12-02T14:20:16,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-12-02T14:20:16,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733149216245, max timestamp=1733149216257Num uniques 1085; 2024-12-02T14:20:16,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733149216245, max timestamp=1733149216257Num uniques 1085; 2024-12-02T14:20:16,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8463, totalHeapSize=19480, min timestamp=1733149216245, max timestamp=1733149216258Num uniques 1085; 2024-12-02T14:20:16,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49683, new segment data size=8463, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-12-02T14:20:16,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-12-02T14:20:16,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-12-02T14:20:16,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733149216277, max timestamp=1733149216282Num uniques 1044; 2024-12-02T14:20:16,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-12-02T14:20:16,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733149216277, max timestamp=1733149216282Num uniques 1044; 2024-12-02T14:20:16,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-12-02T14:20:16,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1733149216277, max timestamp=1733149216282Num uniques 1044; 2024-12-02T14:20:16,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=11174, suffix heap size=136544,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=244 2024-12-02T14:20:16,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149276302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149276304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149276305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149276307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149276307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T14:20:16,409 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:16,409 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:20:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-02T14:20:16,412 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:20:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T14:20:16,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149276419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149276419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149276419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149276419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149276419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,422 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:20:16,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:20:16,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.79 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/6025cb8ac0e344f5b61a9efffdde67fd 2024-12-02T14:20:16,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/0bb79eee86554c2ba513eee36e45a12b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/0bb79eee86554c2ba513eee36e45a12b 2024-12-02T14:20:16,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/0bb79eee86554c2ba513eee36e45a12b, entries=200, sequenceid=162, filesize=14.2 K 2024-12-02T14:20:16,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/11aa8388b4e94318bd8b712d0eb879ce as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/11aa8388b4e94318bd8b712d0eb879ce 2024-12-02T14:20:16,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/11aa8388b4e94318bd8b712d0eb879ce, entries=150, sequenceid=162, filesize=11.9 K 2024-12-02T14:20:16,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/6025cb8ac0e344f5b61a9efffdde67fd as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/6025cb8ac0e344f5b61a9efffdde67fd 2024-12-02T14:20:16,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/6025cb8ac0e344f5b61a9efffdde67fd, entries=150, sequenceid=162, filesize=11.9 K 2024-12-02T14:20:16,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~37.62 KB/38525, heapSize ~95.35 KB/97640, currentSize=147.60 KB/151140 for c7cbc32e95f966c652e56aa6f1b3f54b in 867ms, sequenceid=162, compaction requested=true 2024-12-02T14:20:16,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:16,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:20:16,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:16,469 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:16,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:20:16,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:16,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:20:16,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T14:20:16,469 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:16,471 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:16,471 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/A is initiating minor compaction (all files) 2024-12-02T14:20:16,471 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/A in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:16,471 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/350ee6e94cd54e649fae61db2fdb83fa, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/55e62124507946f3ae39e5ff9199859f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/0bb79eee86554c2ba513eee36e45a12b] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=38.1 K 2024-12-02T14:20:16,477 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:16,477 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/B is initiating minor compaction (all files) 2024-12-02T14:20:16,477 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/B in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:16,477 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/2d27460d1c20424d8f92feb0c777212a, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/cc74ae0df85343c8a0674cbf30c94592, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/11aa8388b4e94318bd8b712d0eb879ce] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=35.8 K 2024-12-02T14:20:16,478 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d27460d1c20424d8f92feb0c777212a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733149211071 2024-12-02T14:20:16,478 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 350ee6e94cd54e649fae61db2fdb83fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733149211071 2024-12-02T14:20:16,478 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting cc74ae0df85343c8a0674cbf30c94592, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733149212249 2024-12-02T14:20:16,479 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55e62124507946f3ae39e5ff9199859f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733149212249 2024-12-02T14:20:16,479 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bb79eee86554c2ba513eee36e45a12b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733149213431 2024-12-02T14:20:16,480 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 11aa8388b4e94318bd8b712d0eb879ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733149213431 2024-12-02T14:20:16,491 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:16,492 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/851b593400b94c57af92d804d133a4aa is 50, key is test_row_0/A:col10/1733149213445/Put/seqid=0 2024-12-02T14:20:16,503 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#B#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:16,503 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/ffab4a5f9e4a4c5c96747a04847890ba is 50, key is test_row_0/B:col10/1733149213445/Put/seqid=0 2024-12-02T14:20:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T14:20:16,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742105_1281 (size=12561) 2024-12-02T14:20:16,545 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/851b593400b94c57af92d804d133a4aa as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/851b593400b94c57af92d804d133a4aa 2024-12-02T14:20:16,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742106_1282 (size=12561) 2024-12-02T14:20:16,554 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/A of c7cbc32e95f966c652e56aa6f1b3f54b into 851b593400b94c57af92d804d133a4aa(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:16,554 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:16,554 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/A, priority=13, startTime=1733149216469; duration=0sec 2024-12-02T14:20:16,554 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:16,554 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:A 2024-12-02T14:20:16,554 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:16,555 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:16,555 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/C is initiating minor compaction (all files) 2024-12-02T14:20:16,555 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/C in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:16,556 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/f944e0425d2a4498a9565686f18d89e3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/39992cdc15c64d95a208a96454caa81b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/6025cb8ac0e344f5b61a9efffdde67fd] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=35.8 K 2024-12-02T14:20:16,556 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting f944e0425d2a4498a9565686f18d89e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733149211071 2024-12-02T14:20:16,556 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39992cdc15c64d95a208a96454caa81b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733149212249 2024-12-02T14:20:16,557 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6025cb8ac0e344f5b61a9efffdde67fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733149213431 2024-12-02T14:20:16,564 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/ffab4a5f9e4a4c5c96747a04847890ba as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/ffab4a5f9e4a4c5c96747a04847890ba 2024-12-02T14:20:16,572 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/B of c7cbc32e95f966c652e56aa6f1b3f54b into ffab4a5f9e4a4c5c96747a04847890ba(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:16,572 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:16,572 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#C#compaction#236 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:16,572 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/B, priority=13, startTime=1733149216469; duration=0sec 2024-12-02T14:20:16,572 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:16,573 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:B 2024-12-02T14:20:16,573 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/b5d9022a27174189a6a79b3e00030584 is 50, key is test_row_0/C:col10/1733149213445/Put/seqid=0 2024-12-02T14:20:16,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T14:20:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:16,576 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=147.60 KB heapSize=416.86 KB 2024-12-02T14:20:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:16,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/1138af321d5e4e9e89bd42af139e671c is 50, key is test_row_0/A:col10/1733149216303/Put/seqid=0 2024-12-02T14:20:16,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742107_1283 (size=12561) 2024-12-02T14:20:16,620 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/b5d9022a27174189a6a79b3e00030584 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/b5d9022a27174189a6a79b3e00030584 2024-12-02T14:20:16,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742108_1284 (size=12301) 2024-12-02T14:20:16,632 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/1138af321d5e4e9e89bd42af139e671c 2024-12-02T14:20:16,632 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/C of c7cbc32e95f966c652e56aa6f1b3f54b into b5d9022a27174189a6a79b3e00030584(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:16,632 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:16,632 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/C, priority=13, startTime=1733149216469; duration=0sec 2024-12-02T14:20:16,632 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:16,632 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:C 2024-12-02T14:20:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:16,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:16,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149276633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149276634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149276634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149276635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149276635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/f9571caf7423495086eb75dc5278c7ff is 50, key is test_row_0/B:col10/1733149216303/Put/seqid=0 2024-12-02T14:20:16,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742109_1285 (size=12301) 2024-12-02T14:20:16,655 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/f9571caf7423495086eb75dc5278c7ff 2024-12-02T14:20:16,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/efbbd76d221c41d59f72d4b702d43918 is 50, key is test_row_0/C:col10/1733149216303/Put/seqid=0 2024-12-02T14:20:16,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742110_1286 (size=12301) 2024-12-02T14:20:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T14:20:16,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149276749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149276749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149276749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149276749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149276749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149276958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149276958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149276959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149276959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149276959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:16,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:16,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:16,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T14:20:17,076 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/efbbd76d221c41d59f72d4b702d43918 2024-12-02T14:20:17,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/1138af321d5e4e9e89bd42af139e671c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/1138af321d5e4e9e89bd42af139e671c 2024-12-02T14:20:17,096 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/1138af321d5e4e9e89bd42af139e671c, entries=150, sequenceid=459, filesize=12.0 K 2024-12-02T14:20:17,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/f9571caf7423495086eb75dc5278c7ff as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/f9571caf7423495086eb75dc5278c7ff 2024-12-02T14:20:17,103 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/f9571caf7423495086eb75dc5278c7ff, entries=150, sequenceid=459, filesize=12.0 K 2024-12-02T14:20:17,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/efbbd76d221c41d59f72d4b702d43918 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/efbbd76d221c41d59f72d4b702d43918 2024-12-02T14:20:17,111 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/efbbd76d221c41d59f72d4b702d43918, entries=150, sequenceid=459, filesize=12.0 K 2024-12-02T14:20:17,112 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~416.81 KB/426816, currentSize=33.54 KB/34350 for c7cbc32e95f966c652e56aa6f1b3f54b in 536ms, sequenceid=459, compaction requested=false 2024-12-02T14:20:17,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:17,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:17,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-02T14:20:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-02T14:20:17,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-02T14:20:17,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 691 msec 2024-12-02T14:20:17,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 710 msec 2024-12-02T14:20:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:17,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T14:20:17,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:17,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:17,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:17,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:17,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:17,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:17,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/266916f36b9347e0a60b3f8d2c493b7e is 50, key is test_row_0/A:col10/1733149217270/Put/seqid=0 2024-12-02T14:20:17,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742111_1287 (size=12301) 2024-12-02T14:20:17,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149217305, max timestamp=1733149217313Num uniques 859; 2024-12-02T14:20:17,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149217305, max timestamp=1733149217313Num uniques 859; 2024-12-02T14:20:17,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:20:17,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:20:17,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149217305, max timestamp=1733149217313Num uniques 859; 2024-12-02T14:20:17,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:20:17,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149277330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149277330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149277334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149277335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149277336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149277438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149277438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149277448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149277448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149277448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T14:20:17,549 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:17,549 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:20:17,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-02T14:20:17,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T14:20:17,551 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:20:17,551 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:20:17,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:20:17,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149277648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149277648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T14:20:17,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149277659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149277659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149277659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/266916f36b9347e0a60b3f8d2c493b7e 2024-12-02T14:20:17,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/f1bd4c32868f47ceadfd96dc5279c2f3 is 50, key is test_row_0/B:col10/1733149217270/Put/seqid=0 2024-12-02T14:20:17,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742112_1288 (size=12301) 2024-12-02T14:20:17,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T14:20:17,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:17,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:17,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:17,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:17,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:17,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:17,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T14:20:17,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:17,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:17,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:17,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:17,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T14:20:17,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149277959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149277959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149277968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149277968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149277968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:18,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T14:20:18,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:18,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:18,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:18,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:18,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:18,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/f1bd4c32868f47ceadfd96dc5279c2f3 2024-12-02T14:20:18,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/092e5b178c364295a3e472234519ec2b is 50, key is test_row_0/C:col10/1733149217270/Put/seqid=0 2024-12-02T14:20:18,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742113_1289 (size=12301) 2024-12-02T14:20:18,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/092e5b178c364295a3e472234519ec2b 2024-12-02T14:20:18,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/266916f36b9347e0a60b3f8d2c493b7e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/266916f36b9347e0a60b3f8d2c493b7e 2024-12-02T14:20:18,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/266916f36b9347e0a60b3f8d2c493b7e, entries=150, sequenceid=470, filesize=12.0 K 2024-12-02T14:20:18,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/f1bd4c32868f47ceadfd96dc5279c2f3 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/f1bd4c32868f47ceadfd96dc5279c2f3 2024-12-02T14:20:18,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/f1bd4c32868f47ceadfd96dc5279c2f3, entries=150, sequenceid=470, filesize=12.0 K 2024-12-02T14:20:18,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/092e5b178c364295a3e472234519ec2b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/092e5b178c364295a3e472234519ec2b 2024-12-02T14:20:18,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/092e5b178c364295a3e472234519ec2b, entries=150, sequenceid=470, filesize=12.0 K 2024-12-02T14:20:18,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for c7cbc32e95f966c652e56aa6f1b3f54b in 880ms, sequenceid=470, compaction requested=true 2024-12-02T14:20:18,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:18,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:20:18,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:18,151 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:18,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:20:18,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:18,151 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:18,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:20:18,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:18,153 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:18,153 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/B is initiating minor compaction (all files) 2024-12-02T14:20:18,153 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/B in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:18,154 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:18,154 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/A is initiating minor compaction (all files) 2024-12-02T14:20:18,154 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/A in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:18,154 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/851b593400b94c57af92d804d133a4aa, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/1138af321d5e4e9e89bd42af139e671c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/266916f36b9347e0a60b3f8d2c493b7e] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=36.3 K 2024-12-02T14:20:18,154 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/ffab4a5f9e4a4c5c96747a04847890ba, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/f9571caf7423495086eb75dc5278c7ff, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/f1bd4c32868f47ceadfd96dc5279c2f3] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=36.3 K 2024-12-02T14:20:18,154 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 851b593400b94c57af92d804d133a4aa, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733149213431 2024-12-02T14:20:18,154 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting ffab4a5f9e4a4c5c96747a04847890ba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733149213431 2024-12-02T14:20:18,155 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1138af321d5e4e9e89bd42af139e671c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733149216297 2024-12-02T14:20:18,155 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting f9571caf7423495086eb75dc5278c7ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733149216297 2024-12-02T14:20:18,155 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 266916f36b9347e0a60b3f8d2c493b7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1733149216630 2024-12-02T14:20:18,155 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting f1bd4c32868f47ceadfd96dc5279c2f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1733149216630 2024-12-02T14:20:18,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T14:20:18,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:18,168 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-02T14:20:18,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:18,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:18,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:18,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:18,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:18,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:18,175 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#A#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:18,175 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:18,176 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/5ec4a0e794174b06b8f6eace0ecd5a33 is 50, key is test_row_0/A:col10/1733149217270/Put/seqid=0 2024-12-02T14:20:18,176 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/8c224c919c8a4beea4a3b44bd6617edc is 50, key is test_row_0/B:col10/1733149217270/Put/seqid=0 2024-12-02T14:20:18,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/5a3544c9fc9748eb990a2b2e4a7d2fc3 is 50, key is test_row_0/A:col10/1733149217328/Put/seqid=0 2024-12-02T14:20:18,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T14:20:18,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742114_1290 (size=12813) 2024-12-02T14:20:18,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742115_1291 (size=12301) 2024-12-02T14:20:18,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742116_1292 (size=12813) 2024-12-02T14:20:18,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:18,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149278479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:18,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:18,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149278479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:18,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:18,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149278479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:18,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149278479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149278480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:18,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149278589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:18,592 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/8c224c919c8a4beea4a3b44bd6617edc as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/8c224c919c8a4beea4a3b44bd6617edc 2024-12-02T14:20:18,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149278589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:18,595 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/5a3544c9fc9748eb990a2b2e4a7d2fc3 2024-12-02T14:20:18,596 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/B of c7cbc32e95f966c652e56aa6f1b3f54b into 8c224c919c8a4beea4a3b44bd6617edc(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:18,596 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:18,596 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/B, priority=13, startTime=1733149218151; duration=0sec 2024-12-02T14:20:18,596 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:18,596 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:B 2024-12-02T14:20:18,596 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:18,597 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:18,598 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/C is initiating minor compaction (all files) 2024-12-02T14:20:18,598 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/C in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:18,598 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/b5d9022a27174189a6a79b3e00030584, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/efbbd76d221c41d59f72d4b702d43918, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/092e5b178c364295a3e472234519ec2b] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=36.3 K 2024-12-02T14:20:18,598 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting b5d9022a27174189a6a79b3e00030584, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733149213431 2024-12-02T14:20:18,598 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting efbbd76d221c41d59f72d4b702d43918, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733149216297 2024-12-02T14:20:18,599 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 092e5b178c364295a3e472234519ec2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1733149216630 2024-12-02T14:20:18,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/5bee67166c3f4c7098ad90ba846d9673 is 50, key is test_row_0/B:col10/1733149217328/Put/seqid=0 2024-12-02T14:20:18,612 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/5ec4a0e794174b06b8f6eace0ecd5a33 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/5ec4a0e794174b06b8f6eace0ecd5a33 2024-12-02T14:20:18,612 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#C#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:18,613 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/7dec2cc13d894b82aabcf05bcaff6d97 is 50, key is test_row_0/C:col10/1733149217270/Put/seqid=0 2024-12-02T14:20:18,618 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/A of c7cbc32e95f966c652e56aa6f1b3f54b into 5ec4a0e794174b06b8f6eace0ecd5a33(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:18,618 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:18,618 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/A, priority=13, startTime=1733149218151; duration=0sec 2024-12-02T14:20:18,618 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:18,618 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:A 2024-12-02T14:20:18,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742117_1293 (size=12301) 2024-12-02T14:20:18,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742118_1294 (size=12813) 2024-12-02T14:20:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T14:20:18,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149278799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:18,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149278799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:19,018 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/5bee67166c3f4c7098ad90ba846d9673 2024-12-02T14:20:19,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/ee3837566a7c4b228d48aa0b9e8053f9 is 50, key is test_row_0/C:col10/1733149217328/Put/seqid=0 2024-12-02T14:20:19,029 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/7dec2cc13d894b82aabcf05bcaff6d97 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/7dec2cc13d894b82aabcf05bcaff6d97 2024-12-02T14:20:19,034 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/C of c7cbc32e95f966c652e56aa6f1b3f54b into 7dec2cc13d894b82aabcf05bcaff6d97(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:19,034 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:19,034 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/C, priority=13, startTime=1733149218151; duration=0sec 2024-12-02T14:20:19,034 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:19,034 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:C 2024-12-02T14:20:19,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742119_1295 (size=12301) 2024-12-02T14:20:19,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149279108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149279108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:19,442 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/ee3837566a7c4b228d48aa0b9e8053f9 2024-12-02T14:20:19,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/5a3544c9fc9748eb990a2b2e4a7d2fc3 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/5a3544c9fc9748eb990a2b2e4a7d2fc3 2024-12-02T14:20:19,449 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/5a3544c9fc9748eb990a2b2e4a7d2fc3, entries=150, sequenceid=507, filesize=12.0 K 2024-12-02T14:20:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/5bee67166c3f4c7098ad90ba846d9673 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5bee67166c3f4c7098ad90ba846d9673 2024-12-02T14:20:19,453 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5bee67166c3f4c7098ad90ba846d9673, entries=150, sequenceid=507, filesize=12.0 K 2024-12-02T14:20:19,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/ee3837566a7c4b228d48aa0b9e8053f9 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/ee3837566a7c4b228d48aa0b9e8053f9 2024-12-02T14:20:19,458 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/ee3837566a7c4b228d48aa0b9e8053f9, entries=150, sequenceid=507, filesize=12.0 K 2024-12-02T14:20:19,458 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for c7cbc32e95f966c652e56aa6f1b3f54b in 1290ms, sequenceid=507, compaction requested=false 2024-12-02T14:20:19,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:19,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:19,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-02T14:20:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-02T14:20:19,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-02T14:20:19,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9090 sec 2024-12-02T14:20:19,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.9130 sec 2024-12-02T14:20:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:19,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T14:20:19,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:19,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:19,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:19,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:19,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:19,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:19,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/0ae1ffe8fdcb44169485007b51fd02bf is 50, key is test_row_0/A:col10/1733149218477/Put/seqid=0 2024-12-02T14:20:19,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742120_1296 (size=14741) 2024-12-02T14:20:19,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/0ae1ffe8fdcb44169485007b51fd02bf 2024-12-02T14:20:19,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/bac9089fa0d3489abfe026ed4e9a4ea6 is 50, key is test_row_0/B:col10/1733149218477/Put/seqid=0 2024-12-02T14:20:19,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742121_1297 (size=12301) 2024-12-02T14:20:19,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149219519, max timestamp=1733149219524Num uniques 859; 2024-12-02T14:20:19,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:20:19,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149219519, max timestamp=1733149219524Num uniques 859; 2024-12-02T14:20:19,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149219519, max timestamp=1733149219524Num uniques 859; 2024-12-02T14:20:19,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:20:19,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:20:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149279546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:19,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149279547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:19,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:19,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149279549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:19,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149279618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:19,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149279618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:19,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149279659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:19,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149279659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:19,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149279659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:19,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T14:20:19,699 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:19,699 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:20:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-02T14:20:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T14:20:19,701 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:20:19,702 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:20:19,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:20:19,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T14:20:19,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T14:20:19,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:19,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:19,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:19,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:19,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:19,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:19,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149279868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:19,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149279868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:19,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:19,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:19,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149279868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:19,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/bac9089fa0d3489abfe026ed4e9a4ea6 2024-12-02T14:20:19,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/50598927a4ef4286a7c2c184bc225c8e is 50, key is test_row_0/C:col10/1733149218477/Put/seqid=0 2024-12-02T14:20:19,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742122_1298 (size=12301) 2024-12-02T14:20:20,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T14:20:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:20,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T14:20:20,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T14:20:20,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:20,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:20,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:20,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:20,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:20,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149280178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:20,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149280178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:20,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:20,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149280179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:20,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:20,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T14:20:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:20,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:20,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T14:20:20,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/50598927a4ef4286a7c2c184bc225c8e 2024-12-02T14:20:20,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/0ae1ffe8fdcb44169485007b51fd02bf as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/0ae1ffe8fdcb44169485007b51fd02bf 2024-12-02T14:20:20,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/0ae1ffe8fdcb44169485007b51fd02bf, entries=200, sequenceid=520, filesize=14.4 K 2024-12-02T14:20:20,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/bac9089fa0d3489abfe026ed4e9a4ea6 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/bac9089fa0d3489abfe026ed4e9a4ea6 2024-12-02T14:20:20,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/bac9089fa0d3489abfe026ed4e9a4ea6, entries=150, sequenceid=520, filesize=12.0 K 2024-12-02T14:20:20,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/50598927a4ef4286a7c2c184bc225c8e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/50598927a4ef4286a7c2c184bc225c8e 2024-12-02T14:20:20,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/50598927a4ef4286a7c2c184bc225c8e, entries=150, sequenceid=520, filesize=12.0 K 2024-12-02T14:20:20,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c7cbc32e95f966c652e56aa6f1b3f54b in 854ms, sequenceid=520, compaction requested=true 2024-12-02T14:20:20,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:20,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:20:20,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:20,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:20:20,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:20,345 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:20,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:20:20,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:20,345 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:20,346 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:20,346 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:20,346 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/B is initiating minor compaction (all files) 2024-12-02T14:20:20,346 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/A is initiating minor compaction (all files) 2024-12-02T14:20:20,346 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/A in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:20,346 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/B in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:20,346 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/5ec4a0e794174b06b8f6eace0ecd5a33, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/5a3544c9fc9748eb990a2b2e4a7d2fc3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/0ae1ffe8fdcb44169485007b51fd02bf] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=38.9 K 2024-12-02T14:20:20,346 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/8c224c919c8a4beea4a3b44bd6617edc, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5bee67166c3f4c7098ad90ba846d9673, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/bac9089fa0d3489abfe026ed4e9a4ea6] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=36.5 K 2024-12-02T14:20:20,346 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c224c919c8a4beea4a3b44bd6617edc, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1733149216630 2024-12-02T14:20:20,346 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ec4a0e794174b06b8f6eace0ecd5a33, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1733149216630 2024-12-02T14:20:20,346 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a3544c9fc9748eb990a2b2e4a7d2fc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1733149217328 2024-12-02T14:20:20,346 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bee67166c3f4c7098ad90ba846d9673, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1733149217328 2024-12-02T14:20:20,347 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting bac9089fa0d3489abfe026ed4e9a4ea6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1733149218473 2024-12-02T14:20:20,347 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ae1ffe8fdcb44169485007b51fd02bf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1733149218473 2024-12-02T14:20:20,356 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:20,356 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/4c530987c3b9493d8bd8698500aa2454 is 50, key is test_row_0/B:col10/1733149218477/Put/seqid=0 2024-12-02T14:20:20,358 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#A#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:20,359 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/00c7626f2de54aca8a5fa94ab8d94cde is 50, key is test_row_0/A:col10/1733149218477/Put/seqid=0 2024-12-02T14:20:20,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742124_1300 (size=12915) 2024-12-02T14:20:20,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742123_1299 (size=12915) 2024-12-02T14:20:20,374 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/4c530987c3b9493d8bd8698500aa2454 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/4c530987c3b9493d8bd8698500aa2454 2024-12-02T14:20:20,379 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/B of c7cbc32e95f966c652e56aa6f1b3f54b into 4c530987c3b9493d8bd8698500aa2454(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:20,379 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:20,379 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/B, priority=13, startTime=1733149220345; duration=0sec 2024-12-02T14:20:20,379 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:20,379 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:B 2024-12-02T14:20:20,379 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:20,380 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:20,380 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/C is initiating minor compaction (all files) 2024-12-02T14:20:20,380 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/C in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:20,380 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/7dec2cc13d894b82aabcf05bcaff6d97, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/ee3837566a7c4b228d48aa0b9e8053f9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/50598927a4ef4286a7c2c184bc225c8e] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=36.5 K 2024-12-02T14:20:20,381 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dec2cc13d894b82aabcf05bcaff6d97, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1733149216630 2024-12-02T14:20:20,381 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting ee3837566a7c4b228d48aa0b9e8053f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1733149217328 2024-12-02T14:20:20,381 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 50598927a4ef4286a7c2c184bc225c8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1733149218473 2024-12-02T14:20:20,388 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#C#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:20,388 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/907f6fd8177c4fe4b9f907050998eb46 is 50, key is test_row_0/C:col10/1733149218477/Put/seqid=0 2024-12-02T14:20:20,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742125_1301 (size=12915) 2024-12-02T14:20:20,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T14:20:20,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:20,468 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T14:20:20,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:20,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:20,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:20,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:20,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:20,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:20,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/b456de53533647d3b89c44725d3bebf8 is 50, key is test_row_0/A:col10/1733149219539/Put/seqid=0 2024-12-02T14:20:20,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742126_1302 (size=12301) 2024-12-02T14:20:20,478 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/b456de53533647d3b89c44725d3bebf8 2024-12-02T14:20:20,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/3a15f0f1fcf643448f7f7ab4c84e8778 is 50, key is test_row_0/B:col10/1733149219539/Put/seqid=0 2024-12-02T14:20:20,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742127_1303 (size=12301) 2024-12-02T14:20:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:20,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:20,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:20,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149280640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:20,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149280640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:20,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149280688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:20,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:20,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149280688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149280688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:20,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:20,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149280748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:20,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149280748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:20,774 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/00c7626f2de54aca8a5fa94ab8d94cde as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/00c7626f2de54aca8a5fa94ab8d94cde 2024-12-02T14:20:20,780 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/A of c7cbc32e95f966c652e56aa6f1b3f54b into 00c7626f2de54aca8a5fa94ab8d94cde(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:20,780 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:20,780 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/A, priority=13, startTime=1733149220345; duration=0sec 2024-12-02T14:20:20,780 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:20,780 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:A 2024-12-02T14:20:20,809 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/907f6fd8177c4fe4b9f907050998eb46 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/907f6fd8177c4fe4b9f907050998eb46 2024-12-02T14:20:20,814 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/C of c7cbc32e95f966c652e56aa6f1b3f54b into 907f6fd8177c4fe4b9f907050998eb46(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:20,814 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:20,814 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/C, priority=13, startTime=1733149220345; duration=0sec 2024-12-02T14:20:20,814 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:20,814 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:C 2024-12-02T14:20:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T14:20:20,890 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/3a15f0f1fcf643448f7f7ab4c84e8778 2024-12-02T14:20:20,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/ce6cdc81379747b19eaa6490e7fe9646 is 50, key is test_row_0/C:col10/1733149219539/Put/seqid=0 2024-12-02T14:20:20,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742128_1304 (size=12301) 2024-12-02T14:20:20,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149280959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149280959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:21,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149281268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149281268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:21,308 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/ce6cdc81379747b19eaa6490e7fe9646 2024-12-02T14:20:21,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/b456de53533647d3b89c44725d3bebf8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b456de53533647d3b89c44725d3bebf8 2024-12-02T14:20:21,317 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b456de53533647d3b89c44725d3bebf8, entries=150, sequenceid=558, filesize=12.0 K 2024-12-02T14:20:21,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/3a15f0f1fcf643448f7f7ab4c84e8778 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/3a15f0f1fcf643448f7f7ab4c84e8778 2024-12-02T14:20:21,324 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/3a15f0f1fcf643448f7f7ab4c84e8778, entries=150, sequenceid=558, filesize=12.0 K 2024-12-02T14:20:21,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/ce6cdc81379747b19eaa6490e7fe9646 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/ce6cdc81379747b19eaa6490e7fe9646 2024-12-02T14:20:21,329 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/ce6cdc81379747b19eaa6490e7fe9646, entries=150, sequenceid=558, filesize=12.0 K 2024-12-02T14:20:21,330 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for c7cbc32e95f966c652e56aa6f1b3f54b in 862ms, sequenceid=558, compaction requested=false 2024-12-02T14:20:21,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:21,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:21,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-02T14:20:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-02T14:20:21,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-02T14:20:21,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6290 sec 2024-12-02T14:20:21,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.6400 sec 2024-12-02T14:20:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:21,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T14:20:21,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:21,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:21,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:21,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:21,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:21,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:21,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/217edee63b534759a2d0061cac229449 is 50, key is test_row_0/A:col10/1733149221699/Put/seqid=0 2024-12-02T14:20:21,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742129_1305 (size=14741) 2024-12-02T14:20:21,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/217edee63b534759a2d0061cac229449 2024-12-02T14:20:21,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/0bdccba495c640a0b80c53a587d675a7 is 50, key is test_row_0/B:col10/1733149221699/Put/seqid=0 2024-12-02T14:20:21,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742130_1306 (size=12301) 2024-12-02T14:20:21,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149281733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:21,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149221721, max timestamp=1733149221735Num uniques 859; 2024-12-02T14:20:21,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:20:21,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149221721, max timestamp=1733149221735Num uniques 859; 2024-12-02T14:20:21,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149221721, max timestamp=1733149221735Num uniques 859; 2024-12-02T14:20:21,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:20:21,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:20:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:21,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149281773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:21,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149281777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:21,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149281778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:21,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149281778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T14:20:21,849 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:21,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149281848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:21,849 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:20:21,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-02T14:20:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T14:20:21,852 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:20:21,858 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:20:21,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:20:21,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149281889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:21,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149281889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T14:20:22,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T14:20:22,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:22,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:22,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:22,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:22,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:22,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149282058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:22,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149282099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:22,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149282099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:22,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/0bdccba495c640a0b80c53a587d675a7 2024-12-02T14:20:22,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/37393577a2b84e94970ba6aa7f50adb8 is 50, key is test_row_0/C:col10/1733149221699/Put/seqid=0 2024-12-02T14:20:22,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742131_1307 (size=12301) 2024-12-02T14:20:22,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T14:20:22,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:22,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:22,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:22,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:22,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T14:20:22,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T14:20:22,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:22,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:22,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:22,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:22,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:22,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149282368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:22,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:22,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149282408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:22,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149282408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:22,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T14:20:22,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:22,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:22,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:22,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:22,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T14:20:22,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/37393577a2b84e94970ba6aa7f50adb8 2024-12-02T14:20:22,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/217edee63b534759a2d0061cac229449 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/217edee63b534759a2d0061cac229449 2024-12-02T14:20:22,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/217edee63b534759a2d0061cac229449, entries=200, sequenceid=571, filesize=14.4 K 2024-12-02T14:20:22,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/0bdccba495c640a0b80c53a587d675a7 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/0bdccba495c640a0b80c53a587d675a7 2024-12-02T14:20:22,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/0bdccba495c640a0b80c53a587d675a7, entries=150, sequenceid=571, filesize=12.0 K 2024-12-02T14:20:22,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/37393577a2b84e94970ba6aa7f50adb8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/37393577a2b84e94970ba6aa7f50adb8 2024-12-02T14:20:22,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/37393577a2b84e94970ba6aa7f50adb8, entries=150, sequenceid=571, filesize=12.0 K 2024-12-02T14:20:22,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c7cbc32e95f966c652e56aa6f1b3f54b in 874ms, sequenceid=571, compaction requested=true 2024-12-02T14:20:22,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:22,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:20:22,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:22,574 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:22,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:20:22,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:22,574 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:22,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:20:22,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:22,575 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:22,575 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:22,575 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/A is initiating minor compaction (all files) 2024-12-02T14:20:22,575 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/B is initiating minor compaction (all files) 2024-12-02T14:20:22,575 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/B in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:22,575 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/A in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:22,575 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/00c7626f2de54aca8a5fa94ab8d94cde, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b456de53533647d3b89c44725d3bebf8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/217edee63b534759a2d0061cac229449] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=39.0 K 2024-12-02T14:20:22,575 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/4c530987c3b9493d8bd8698500aa2454, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/3a15f0f1fcf643448f7f7ab4c84e8778, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/0bdccba495c640a0b80c53a587d675a7] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=36.6 K 2024-12-02T14:20:22,576 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00c7626f2de54aca8a5fa94ab8d94cde, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1733149218473 2024-12-02T14:20:22,576 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c530987c3b9493d8bd8698500aa2454, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1733149218473 2024-12-02T14:20:22,576 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting b456de53533647d3b89c44725d3bebf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1733149219539 2024-12-02T14:20:22,576 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a15f0f1fcf643448f7f7ab4c84e8778, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1733149219539 2024-12-02T14:20:22,576 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 217edee63b534759a2d0061cac229449, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1733149220637 2024-12-02T14:20:22,576 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bdccba495c640a0b80c53a587d675a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1733149220637 2024-12-02T14:20:22,581 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#A#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:22,582 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/150db3d714c9496e937ae661f87908ac is 50, key is test_row_0/A:col10/1733149221699/Put/seqid=0 2024-12-02T14:20:22,582 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#B#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:22,583 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/5f6e7277323a4899b45cf3569865f7f6 is 50, key is test_row_0/B:col10/1733149221699/Put/seqid=0 2024-12-02T14:20:22,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742132_1308 (size=13017) 2024-12-02T14:20:22,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742133_1309 (size=13017) 2024-12-02T14:20:22,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T14:20:22,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:22,626 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-02T14:20:22,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:22,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:22,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:22,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:22,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:22,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:22,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/8935438d1b0342f9b33e9f6639d3c1a3 is 50, key is test_row_0/A:col10/1733149221761/Put/seqid=0 2024-12-02T14:20:22,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742134_1310 (size=12301) 2024-12-02T14:20:22,633 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/8935438d1b0342f9b33e9f6639d3c1a3 2024-12-02T14:20:22,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/aafce8fd72ec4af09eab6af615738acf is 50, key is test_row_0/B:col10/1733149221761/Put/seqid=0 2024-12-02T14:20:22,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742135_1311 (size=12301) 2024-12-02T14:20:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:22,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:22,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149282816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:22,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149282816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:22,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149282879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:22,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:22,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149282918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:22,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:22,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149282929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149282928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:22,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149282929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T14:20:22,992 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/150db3d714c9496e937ae661f87908ac as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/150db3d714c9496e937ae661f87908ac 2024-12-02T14:20:22,997 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/A of c7cbc32e95f966c652e56aa6f1b3f54b into 150db3d714c9496e937ae661f87908ac(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:22,997 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:22,997 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/A, priority=13, startTime=1733149222574; duration=0sec 2024-12-02T14:20:22,997 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:22,997 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:A 2024-12-02T14:20:22,997 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:23,006 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/5f6e7277323a4899b45cf3569865f7f6 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5f6e7277323a4899b45cf3569865f7f6 2024-12-02T14:20:23,006 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:23,006 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/C is initiating minor compaction (all files) 2024-12-02T14:20:23,006 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/C in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:23,007 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/907f6fd8177c4fe4b9f907050998eb46, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/ce6cdc81379747b19eaa6490e7fe9646, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/37393577a2b84e94970ba6aa7f50adb8] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=36.6 K 2024-12-02T14:20:23,007 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 907f6fd8177c4fe4b9f907050998eb46, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1733149218473 2024-12-02T14:20:23,007 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce6cdc81379747b19eaa6490e7fe9646, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=558, earliestPutTs=1733149219539 2024-12-02T14:20:23,008 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37393577a2b84e94970ba6aa7f50adb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1733149220637 2024-12-02T14:20:23,011 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/B of c7cbc32e95f966c652e56aa6f1b3f54b into 5f6e7277323a4899b45cf3569865f7f6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:23,011 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:23,011 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/B, priority=13, startTime=1733149222574; duration=0sec 2024-12-02T14:20:23,011 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:23,011 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:B 2024-12-02T14:20:23,018 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#C#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:23,019 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/9d0269a11ec247d296335fa24993aa92 is 50, key is test_row_0/C:col10/1733149221699/Put/seqid=0 2024-12-02T14:20:23,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742136_1312 (size=13017) 2024-12-02T14:20:23,044 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/aafce8fd72ec4af09eab6af615738acf 2024-12-02T14:20:23,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/e22becfa7de248a4a8f35efe9b539456 is 50, key is test_row_0/C:col10/1733149221761/Put/seqid=0 2024-12-02T14:20:23,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742137_1313 (size=12301) 2024-12-02T14:20:23,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149283138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149283139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:23,427 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/9d0269a11ec247d296335fa24993aa92 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/9d0269a11ec247d296335fa24993aa92 2024-12-02T14:20:23,431 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/C of c7cbc32e95f966c652e56aa6f1b3f54b into 9d0269a11ec247d296335fa24993aa92(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:23,431 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:23,431 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/C, priority=13, startTime=1733149222574; duration=0sec 2024-12-02T14:20:23,431 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:23,431 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:C 2024-12-02T14:20:23,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149283448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:23,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149283448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:23,454 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/e22becfa7de248a4a8f35efe9b539456 2024-12-02T14:20:23,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/8935438d1b0342f9b33e9f6639d3c1a3 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/8935438d1b0342f9b33e9f6639d3c1a3 2024-12-02T14:20:23,461 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/8935438d1b0342f9b33e9f6639d3c1a3, entries=150, sequenceid=607, filesize=12.0 K 2024-12-02T14:20:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/aafce8fd72ec4af09eab6af615738acf as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/aafce8fd72ec4af09eab6af615738acf 2024-12-02T14:20:23,465 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/aafce8fd72ec4af09eab6af615738acf, entries=150, sequenceid=607, filesize=12.0 K 2024-12-02T14:20:23,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/e22becfa7de248a4a8f35efe9b539456 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/e22becfa7de248a4a8f35efe9b539456 2024-12-02T14:20:23,479 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/e22becfa7de248a4a8f35efe9b539456, entries=150, sequenceid=607, filesize=12.0 K 2024-12-02T14:20:23,480 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for c7cbc32e95f966c652e56aa6f1b3f54b in 854ms, sequenceid=607, compaction requested=false 2024-12-02T14:20:23,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:23,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:23,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-02T14:20:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-02T14:20:23,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-02T14:20:23,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6230 sec 2024-12-02T14:20:23,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.6340 sec 2024-12-02T14:20:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:23,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T14:20:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:23,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/66ead0f5693d4a2bac50b0d6f20ac2fc is 50, key is test_row_0/A:col10/1733149222814/Put/seqid=0 2024-12-02T14:20:23,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742138_1314 (size=12301) 2024-12-02T14:20:23,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149283943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:23,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:23,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149283946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149283946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:23,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:23,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149283958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149283958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T14:20:23,999 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:23,999 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:20:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-02T14:20:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T14:20:24,001 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:20:24,001 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:20:24,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:20:24,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149284048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:24,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149284058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149284058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T14:20:24,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T14:20:24,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:24,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:24,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:24,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149284259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:24,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149284268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149284268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:24,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/66ead0f5693d4a2bac50b0d6f20ac2fc 2024-12-02T14:20:24,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T14:20:24,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:24,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:24,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:24,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/8122eb2e4b9b4770817fac38a2a6bb4c is 50, key is test_row_0/B:col10/1733149222814/Put/seqid=0 2024-12-02T14:20:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742139_1315 (size=12301) 2024-12-02T14:20:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T14:20:24,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T14:20:24,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:24,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:24,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:24,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149284569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:24,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149284578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:24,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149284578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:24,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T14:20:24,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:24,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:24,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:24,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T14:20:24,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/8122eb2e4b9b4770817fac38a2a6bb4c 2024-12-02T14:20:24,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/5e0b624aff4f40edae242f4054eaabe4 is 50, key is test_row_0/C:col10/1733149222814/Put/seqid=0 2024-12-02T14:20:24,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742140_1316 (size=12301) 2024-12-02T14:20:24,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T14:20:24,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:24,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:24,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:24,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T14:20:24,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:24,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:24,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:24,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:24,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1733149284968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:24,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1733149284968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:24,971 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in c7cbc32e95f966c652e56aa6f1b3f54b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:24,971 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_1' in c7cbc32e95f966c652e56aa6f1b3f54b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:24,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:25,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T14:20:25,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:25,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:25,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:25,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:25,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:25,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1733149285078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:25,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45164 deadline: 1733149285088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:25,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:25,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45194 deadline: 1733149285088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:20:25,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/5e0b624aff4f40edae242f4054eaabe4 2024-12-02T14:20:25,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/66ead0f5693d4a2bac50b0d6f20ac2fc as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/66ead0f5693d4a2bac50b0d6f20ac2fc 2024-12-02T14:20:25,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/66ead0f5693d4a2bac50b0d6f20ac2fc, entries=150, sequenceid=623, filesize=12.0 K 2024-12-02T14:20:25,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/8122eb2e4b9b4770817fac38a2a6bb4c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/8122eb2e4b9b4770817fac38a2a6bb4c 2024-12-02T14:20:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T14:20:25,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/8122eb2e4b9b4770817fac38a2a6bb4c, entries=150, sequenceid=623, filesize=12.0 K 2024-12-02T14:20:25,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/5e0b624aff4f40edae242f4054eaabe4 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/5e0b624aff4f40edae242f4054eaabe4 2024-12-02T14:20:25,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/5e0b624aff4f40edae242f4054eaabe4, entries=150, sequenceid=623, filesize=12.0 K 2024-12-02T14:20:25,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c7cbc32e95f966c652e56aa6f1b3f54b in 1254ms, sequenceid=623, compaction requested=true 2024-12-02T14:20:25,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:25,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:20:25,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:25,144 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:25,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:20:25,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:25,144 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:25,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7cbc32e95f966c652e56aa6f1b3f54b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:20:25,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:25,145 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:25,145 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/A is initiating minor compaction (all files) 2024-12-02T14:20:25,145 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/A in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:25,146 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/150db3d714c9496e937ae661f87908ac, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/8935438d1b0342f9b33e9f6639d3c1a3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/66ead0f5693d4a2bac50b0d6f20ac2fc] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=36.7 K 2024-12-02T14:20:25,146 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:25,146 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/B is initiating minor compaction (all files) 2024-12-02T14:20:25,146 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 150db3d714c9496e937ae661f87908ac, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1733149220637 2024-12-02T14:20:25,146 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/B in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:25,146 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5f6e7277323a4899b45cf3569865f7f6, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/aafce8fd72ec4af09eab6af615738acf, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/8122eb2e4b9b4770817fac38a2a6bb4c] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=36.7 K 2024-12-02T14:20:25,146 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8935438d1b0342f9b33e9f6639d3c1a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1733149221759 2024-12-02T14:20:25,146 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f6e7277323a4899b45cf3569865f7f6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1733149220637 2024-12-02T14:20:25,146 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66ead0f5693d4a2bac50b0d6f20ac2fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=623, earliestPutTs=1733149222807 2024-12-02T14:20:25,147 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting aafce8fd72ec4af09eab6af615738acf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1733149221759 2024-12-02T14:20:25,147 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 8122eb2e4b9b4770817fac38a2a6bb4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=623, earliestPutTs=1733149222807 2024-12-02T14:20:25,152 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#B#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:25,153 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/0df906f256da46658b4045e47b1cd593 is 50, key is test_row_0/B:col10/1733149222814/Put/seqid=0 2024-12-02T14:20:25,153 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#A#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:25,153 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/e4b7b2f8fb2f4ad586c2ed0b7c4555ef is 50, key is test_row_0/A:col10/1733149222814/Put/seqid=0 2024-12-02T14:20:25,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742142_1318 (size=13119) 2024-12-02T14:20:25,161 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/e4b7b2f8fb2f4ad586c2ed0b7c4555ef as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/e4b7b2f8fb2f4ad586c2ed0b7c4555ef 2024-12-02T14:20:25,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742141_1317 (size=13119) 2024-12-02T14:20:25,167 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/A of c7cbc32e95f966c652e56aa6f1b3f54b into e4b7b2f8fb2f4ad586c2ed0b7c4555ef(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:25,168 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:25,168 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/A, priority=13, startTime=1733149225144; duration=0sec 2024-12-02T14:20:25,168 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:25,168 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:A 2024-12-02T14:20:25,168 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:25,169 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:25,169 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): c7cbc32e95f966c652e56aa6f1b3f54b/C is initiating minor compaction (all files) 2024-12-02T14:20:25,169 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7cbc32e95f966c652e56aa6f1b3f54b/C in TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:25,169 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/9d0269a11ec247d296335fa24993aa92, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/e22becfa7de248a4a8f35efe9b539456, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/5e0b624aff4f40edae242f4054eaabe4] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp, totalSize=36.7 K 2024-12-02T14:20:25,169 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d0269a11ec247d296335fa24993aa92, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1733149220637 2024-12-02T14:20:25,169 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting e22becfa7de248a4a8f35efe9b539456, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1733149221759 2024-12-02T14:20:25,170 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e0b624aff4f40edae242f4054eaabe4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=623, earliestPutTs=1733149222807 2024-12-02T14:20:25,177 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7cbc32e95f966c652e56aa6f1b3f54b#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:25,177 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/1d6de0338ab24714a93fb78b50faa32e is 50, key is test_row_0/C:col10/1733149222814/Put/seqid=0 2024-12-02T14:20:25,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742143_1319 (size=13119) 2024-12-02T14:20:25,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T14:20:25,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:25,227 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T14:20:25,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:25,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:25,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:25,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:25,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:25,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:25,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/9b02d160c68f4082891d75050b34c345 is 50, key is test_row_0/A:col10/1733149223942/Put/seqid=0 2024-12-02T14:20:25,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742144_1320 (size=12301) 2024-12-02T14:20:25,572 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/0df906f256da46658b4045e47b1cd593 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/0df906f256da46658b4045e47b1cd593 2024-12-02T14:20:25,576 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/B of c7cbc32e95f966c652e56aa6f1b3f54b into 0df906f256da46658b4045e47b1cd593(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:25,576 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:25,576 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/B, priority=13, startTime=1733149225144; duration=0sec 2024-12-02T14:20:25,576 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:25,576 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:B 2024-12-02T14:20:25,613 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/1d6de0338ab24714a93fb78b50faa32e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/1d6de0338ab24714a93fb78b50faa32e 2024-12-02T14:20:25,617 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7cbc32e95f966c652e56aa6f1b3f54b/C of c7cbc32e95f966c652e56aa6f1b3f54b into 1d6de0338ab24714a93fb78b50faa32e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:25,617 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:25,617 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., storeName=c7cbc32e95f966c652e56aa6f1b3f54b/C, priority=13, startTime=1733149225144; duration=0sec 2024-12-02T14:20:25,617 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:25,617 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7cbc32e95f966c652e56aa6f1b3f54b:C 2024-12-02T14:20:25,672 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/9b02d160c68f4082891d75050b34c345 2024-12-02T14:20:25,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/436bc90104f94336ac6367e5fa9b5d28 is 50, key is test_row_0/B:col10/1733149223942/Put/seqid=0 2024-12-02T14:20:25,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742145_1321 (size=12301) 2024-12-02T14:20:25,792 INFO [Thread-1145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1145. 2024-12-02T14:20:25,792 DEBUG [Thread-1145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:25,792 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:25,792 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:25,792 INFO [Thread-1147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1147. 2024-12-02T14:20:25,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:25,792 DEBUG [Thread-1147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:25,792 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:25,792 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:25,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:25,793 INFO [Thread-1151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1151. 2024-12-02T14:20:25,793 DEBUG [Thread-1151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:25,793 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:25,793 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:25,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:25,794 INFO [Thread-1153 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1153. 2024-12-02T14:20:25,794 DEBUG [Thread-1153 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:25,794 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:25,794 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:25,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:25,796 INFO [Thread-1149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1149. 2024-12-02T14:20:25,796 DEBUG [Thread-1149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:25,796 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:25,796 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:25,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:26,084 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/436bc90104f94336ac6367e5fa9b5d28 2024-12-02T14:20:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:26,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. as already flushing 2024-12-02T14:20:26,092 INFO [Thread-1142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1142. 2024-12-02T14:20:26,092 DEBUG [Thread-1142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:26,092 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:26,092 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:26,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:26,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/283e2a5368754d70afdc5af356c01492 is 50, key is test_row_0/C:col10/1733149223942/Put/seqid=0 2024-12-02T14:20:26,101 INFO [Thread-1136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1136. 2024-12-02T14:20:26,101 INFO [Thread-1138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1138. 2024-12-02T14:20:26,101 DEBUG [Thread-1138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:26,101 DEBUG [Thread-1136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:26,101 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:26,101 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:26,101 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:26,101 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:26,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:26,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:26,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742146_1322 (size=12301) 2024-12-02T14:20:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T14:20:26,503 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/283e2a5368754d70afdc5af356c01492 2024-12-02T14:20:26,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/9b02d160c68f4082891d75050b34c345 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/9b02d160c68f4082891d75050b34c345 2024-12-02T14:20:26,518 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/9b02d160c68f4082891d75050b34c345, entries=150, sequenceid=643, filesize=12.0 K 2024-12-02T14:20:26,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/436bc90104f94336ac6367e5fa9b5d28 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/436bc90104f94336ac6367e5fa9b5d28 2024-12-02T14:20:26,523 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/436bc90104f94336ac6367e5fa9b5d28, entries=150, sequenceid=643, filesize=12.0 K 2024-12-02T14:20:26,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/283e2a5368754d70afdc5af356c01492 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/283e2a5368754d70afdc5af356c01492 2024-12-02T14:20:26,527 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/283e2a5368754d70afdc5af356c01492, entries=150, sequenceid=643, filesize=12.0 K 2024-12-02T14:20:26,528 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=20.13 KB/20610 for c7cbc32e95f966c652e56aa6f1b3f54b in 1301ms, sequenceid=643, compaction requested=false 2024-12-02T14:20:26,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for c7cbc32e95f966c652e56aa6f1b3f54b: 2024-12-02T14:20:26,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:26,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-02T14:20:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-02T14:20:26,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-02T14:20:26,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5270 sec 2024-12-02T14:20:26,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.5320 sec 2024-12-02T14:20:26,981 INFO [Thread-1134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1134. 2024-12-02T14:20:26,981 DEBUG [Thread-1134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:26,981 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:26,981 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:26,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:26,991 INFO [Thread-1140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1140. 2024-12-02T14:20:26,991 DEBUG [Thread-1140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:26,991 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:26,991 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:26,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:27,047 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T14:20:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T14:20:28,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T14:20:28,169 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:28,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-12-02T14:20:28,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-12-02T14:20:28,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-12-02T14:20:28,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 136 2024-12-02T14:20:28,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-12-02T14:20:28,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T14:20:28,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4133 2024-12-02T14:20:28,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4223 2024-12-02T14:20:28,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4174 2024-12-02T14:20:28,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4112 2024-12-02T14:20:28,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4210 2024-12-02T14:20:28,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T14:20:28,169 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T14:20:28,169 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T14:20:28,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:28,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:28,170 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:28,170 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T14:20:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T14:20:28,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149228174"}]},"ts":"1733149228174"} 2024-12-02T14:20:28,175 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T14:20:28,176 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T14:20:28,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T14:20:28,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7cbc32e95f966c652e56aa6f1b3f54b, UNASSIGN}] 2024-12-02T14:20:28,178 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7cbc32e95f966c652e56aa6f1b3f54b, UNASSIGN 2024-12-02T14:20:28,179 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=c7cbc32e95f966c652e56aa6f1b3f54b, regionState=CLOSING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:28,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7cbc32e95f966c652e56aa6f1b3f54b, UNASSIGN because future has completed 2024-12-02T14:20:28,181 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T14:20:28,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:20:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T14:20:28,334 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:28,334 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T14:20:28,334 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing c7cbc32e95f966c652e56aa6f1b3f54b, disabling compactions & flushes 2024-12-02T14:20:28,334 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:28,334 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:28,334 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. after waiting 0 ms 2024-12-02T14:20:28,334 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:28,334 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing c7cbc32e95f966c652e56aa6f1b3f54b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T14:20:28,334 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=A 2024-12-02T14:20:28,334 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:28,334 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=B 2024-12-02T14:20:28,334 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:28,334 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7cbc32e95f966c652e56aa6f1b3f54b, store=C 2024-12-02T14:20:28,334 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:28,338 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/86d4fbb1863742de908aebd101975732 is 50, key is test_row_0/A:col10/1733149226979/Put/seqid=0 2024-12-02T14:20:28,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742147_1323 (size=12301) 2024-12-02T14:20:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T14:20:28,754 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/86d4fbb1863742de908aebd101975732 2024-12-02T14:20:28,760 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/56ec88e2f7e448388630f1d0c392ed1c is 50, key is test_row_0/B:col10/1733149226979/Put/seqid=0 2024-12-02T14:20:28,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742148_1324 (size=12301) 2024-12-02T14:20:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T14:20:29,163 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/56ec88e2f7e448388630f1d0c392ed1c 2024-12-02T14:20:29,170 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/c5629d995aa441bd9632d61a16334021 is 50, key is test_row_0/C:col10/1733149226979/Put/seqid=0 2024-12-02T14:20:29,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742149_1325 (size=12301) 2024-12-02T14:20:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T14:20:29,592 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/c5629d995aa441bd9632d61a16334021 2024-12-02T14:20:29,600 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/A/86d4fbb1863742de908aebd101975732 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/86d4fbb1863742de908aebd101975732 2024-12-02T14:20:29,605 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/86d4fbb1863742de908aebd101975732, entries=150, sequenceid=653, filesize=12.0 K 2024-12-02T14:20:29,606 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/B/56ec88e2f7e448388630f1d0c392ed1c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/56ec88e2f7e448388630f1d0c392ed1c 2024-12-02T14:20:29,610 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/56ec88e2f7e448388630f1d0c392ed1c, entries=150, sequenceid=653, filesize=12.0 K 2024-12-02T14:20:29,611 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/.tmp/C/c5629d995aa441bd9632d61a16334021 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/c5629d995aa441bd9632d61a16334021 2024-12-02T14:20:29,617 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/c5629d995aa441bd9632d61a16334021, entries=150, sequenceid=653, filesize=12.0 K 2024-12-02T14:20:29,618 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c7cbc32e95f966c652e56aa6f1b3f54b in 1284ms, sequenceid=653, compaction requested=true 2024-12-02T14:20:29,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/9efb5e8636da44848291c308c68e63de, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/820e96cd564f4a689a4f0ed6c3ea4d58, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/c38510a098bf4bcc8a4693185f7ab13c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b074995bbf2648068172f71bd22849a4, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/01d2a7e995c948c2ac8133cb4bc222b7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/076e19747e274b36a942fc8d901a659b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/c36808c25602430785413c448579fb3f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/12881aa8919d44a787b077e8ebf3e10d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/350ee6e94cd54e649fae61db2fdb83fa, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b8d49a4cc7b2451aae5668c36c1709b3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/55e62124507946f3ae39e5ff9199859f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/0bb79eee86554c2ba513eee36e45a12b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/851b593400b94c57af92d804d133a4aa, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/1138af321d5e4e9e89bd42af139e671c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/5ec4a0e794174b06b8f6eace0ecd5a33, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/266916f36b9347e0a60b3f8d2c493b7e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/5a3544c9fc9748eb990a2b2e4a7d2fc3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/0ae1ffe8fdcb44169485007b51fd02bf, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/00c7626f2de54aca8a5fa94ab8d94cde, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b456de53533647d3b89c44725d3bebf8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/217edee63b534759a2d0061cac229449, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/150db3d714c9496e937ae661f87908ac, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/8935438d1b0342f9b33e9f6639d3c1a3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/66ead0f5693d4a2bac50b0d6f20ac2fc] to archive 2024-12-02T14:20:29,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:20:29,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/9efb5e8636da44848291c308c68e63de to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/9efb5e8636da44848291c308c68e63de 2024-12-02T14:20:29,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/820e96cd564f4a689a4f0ed6c3ea4d58 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/820e96cd564f4a689a4f0ed6c3ea4d58 2024-12-02T14:20:29,625 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/c38510a098bf4bcc8a4693185f7ab13c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/c38510a098bf4bcc8a4693185f7ab13c 2024-12-02T14:20:29,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b074995bbf2648068172f71bd22849a4 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b074995bbf2648068172f71bd22849a4 2024-12-02T14:20:29,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/01d2a7e995c948c2ac8133cb4bc222b7 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/01d2a7e995c948c2ac8133cb4bc222b7 2024-12-02T14:20:29,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/076e19747e274b36a942fc8d901a659b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/076e19747e274b36a942fc8d901a659b 2024-12-02T14:20:29,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/c36808c25602430785413c448579fb3f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/c36808c25602430785413c448579fb3f 2024-12-02T14:20:29,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/12881aa8919d44a787b077e8ebf3e10d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/12881aa8919d44a787b077e8ebf3e10d 2024-12-02T14:20:29,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/350ee6e94cd54e649fae61db2fdb83fa to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/350ee6e94cd54e649fae61db2fdb83fa 2024-12-02T14:20:29,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b8d49a4cc7b2451aae5668c36c1709b3 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b8d49a4cc7b2451aae5668c36c1709b3 2024-12-02T14:20:29,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/55e62124507946f3ae39e5ff9199859f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/55e62124507946f3ae39e5ff9199859f 2024-12-02T14:20:29,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/0bb79eee86554c2ba513eee36e45a12b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/0bb79eee86554c2ba513eee36e45a12b 2024-12-02T14:20:29,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/851b593400b94c57af92d804d133a4aa to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/851b593400b94c57af92d804d133a4aa 2024-12-02T14:20:29,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/1138af321d5e4e9e89bd42af139e671c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/1138af321d5e4e9e89bd42af139e671c 2024-12-02T14:20:29,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/5ec4a0e794174b06b8f6eace0ecd5a33 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/5ec4a0e794174b06b8f6eace0ecd5a33 2024-12-02T14:20:29,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/266916f36b9347e0a60b3f8d2c493b7e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/266916f36b9347e0a60b3f8d2c493b7e 2024-12-02T14:20:29,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/5a3544c9fc9748eb990a2b2e4a7d2fc3 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/5a3544c9fc9748eb990a2b2e4a7d2fc3 2024-12-02T14:20:29,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/0ae1ffe8fdcb44169485007b51fd02bf to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/0ae1ffe8fdcb44169485007b51fd02bf 2024-12-02T14:20:29,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/00c7626f2de54aca8a5fa94ab8d94cde to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/00c7626f2de54aca8a5fa94ab8d94cde 2024-12-02T14:20:29,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b456de53533647d3b89c44725d3bebf8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/b456de53533647d3b89c44725d3bebf8 2024-12-02T14:20:29,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/217edee63b534759a2d0061cac229449 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/217edee63b534759a2d0061cac229449 2024-12-02T14:20:29,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/150db3d714c9496e937ae661f87908ac to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/150db3d714c9496e937ae661f87908ac 2024-12-02T14:20:29,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/8935438d1b0342f9b33e9f6639d3c1a3 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/8935438d1b0342f9b33e9f6639d3c1a3 2024-12-02T14:20:29,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/66ead0f5693d4a2bac50b0d6f20ac2fc to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/66ead0f5693d4a2bac50b0d6f20ac2fc 2024-12-02T14:20:29,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/3c4f3b6b16cc449487d28bced1556f6c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/c71dac672e0940bcb5fc31c85f410b6f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/a42b8e3bc9424827bd5e7cbf969ab9d6, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/518c73c821a04538b3ed55ba4b80d7e8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/44b6283560bf4e95a4bc81aaa1cfaeb5, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/e90306f41fca4abdae0e847fed1a97c2, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5b645633dac94a8ca2d065ac990d40b2, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/584e980d43284412b9cdb2ef21d3c89c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/2d27460d1c20424d8f92feb0c777212a, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/d57a454a8e714202a73451e0eeddfdca, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/cc74ae0df85343c8a0674cbf30c94592, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/ffab4a5f9e4a4c5c96747a04847890ba, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/11aa8388b4e94318bd8b712d0eb879ce, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/f9571caf7423495086eb75dc5278c7ff, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/8c224c919c8a4beea4a3b44bd6617edc, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/f1bd4c32868f47ceadfd96dc5279c2f3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5bee67166c3f4c7098ad90ba846d9673, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/4c530987c3b9493d8bd8698500aa2454, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/bac9089fa0d3489abfe026ed4e9a4ea6, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/3a15f0f1fcf643448f7f7ab4c84e8778, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5f6e7277323a4899b45cf3569865f7f6, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/0bdccba495c640a0b80c53a587d675a7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/aafce8fd72ec4af09eab6af615738acf, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/8122eb2e4b9b4770817fac38a2a6bb4c] to archive 2024-12-02T14:20:29,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:20:29,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/3c4f3b6b16cc449487d28bced1556f6c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/3c4f3b6b16cc449487d28bced1556f6c 2024-12-02T14:20:29,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/c71dac672e0940bcb5fc31c85f410b6f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/c71dac672e0940bcb5fc31c85f410b6f 2024-12-02T14:20:29,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/a42b8e3bc9424827bd5e7cbf969ab9d6 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/a42b8e3bc9424827bd5e7cbf969ab9d6 2024-12-02T14:20:29,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/518c73c821a04538b3ed55ba4b80d7e8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/518c73c821a04538b3ed55ba4b80d7e8 2024-12-02T14:20:29,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/44b6283560bf4e95a4bc81aaa1cfaeb5 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/44b6283560bf4e95a4bc81aaa1cfaeb5 2024-12-02T14:20:29,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/e90306f41fca4abdae0e847fed1a97c2 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/e90306f41fca4abdae0e847fed1a97c2 2024-12-02T14:20:29,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5b645633dac94a8ca2d065ac990d40b2 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5b645633dac94a8ca2d065ac990d40b2 2024-12-02T14:20:29,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/584e980d43284412b9cdb2ef21d3c89c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/584e980d43284412b9cdb2ef21d3c89c 2024-12-02T14:20:29,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/2d27460d1c20424d8f92feb0c777212a to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/2d27460d1c20424d8f92feb0c777212a 2024-12-02T14:20:29,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/d57a454a8e714202a73451e0eeddfdca to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/d57a454a8e714202a73451e0eeddfdca 2024-12-02T14:20:29,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/cc74ae0df85343c8a0674cbf30c94592 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/cc74ae0df85343c8a0674cbf30c94592 2024-12-02T14:20:29,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/ffab4a5f9e4a4c5c96747a04847890ba to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/ffab4a5f9e4a4c5c96747a04847890ba 2024-12-02T14:20:29,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/11aa8388b4e94318bd8b712d0eb879ce to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/11aa8388b4e94318bd8b712d0eb879ce 2024-12-02T14:20:29,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/f9571caf7423495086eb75dc5278c7ff to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/f9571caf7423495086eb75dc5278c7ff 2024-12-02T14:20:29,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/8c224c919c8a4beea4a3b44bd6617edc to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/8c224c919c8a4beea4a3b44bd6617edc 2024-12-02T14:20:29,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/f1bd4c32868f47ceadfd96dc5279c2f3 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/f1bd4c32868f47ceadfd96dc5279c2f3 2024-12-02T14:20:29,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5bee67166c3f4c7098ad90ba846d9673 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5bee67166c3f4c7098ad90ba846d9673 2024-12-02T14:20:29,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/4c530987c3b9493d8bd8698500aa2454 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/4c530987c3b9493d8bd8698500aa2454 2024-12-02T14:20:29,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/bac9089fa0d3489abfe026ed4e9a4ea6 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/bac9089fa0d3489abfe026ed4e9a4ea6 2024-12-02T14:20:29,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/3a15f0f1fcf643448f7f7ab4c84e8778 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/3a15f0f1fcf643448f7f7ab4c84e8778 2024-12-02T14:20:29,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5f6e7277323a4899b45cf3569865f7f6 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/5f6e7277323a4899b45cf3569865f7f6 2024-12-02T14:20:29,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/0bdccba495c640a0b80c53a587d675a7 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/0bdccba495c640a0b80c53a587d675a7 2024-12-02T14:20:29,674 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/aafce8fd72ec4af09eab6af615738acf to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/aafce8fd72ec4af09eab6af615738acf 2024-12-02T14:20:29,675 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/8122eb2e4b9b4770817fac38a2a6bb4c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/8122eb2e4b9b4770817fac38a2a6bb4c 2024-12-02T14:20:29,676 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/a3231e9cd78b43398b03349ed9795327, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/c64b06c0db964977a2eecb128bf69a2b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/aa63f39833d24ec4b09ab3f6e0521433, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/bbbd7619e0be4182bcb432d314f7bebe, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/e80c7a83f1f64d17a370e777c7979497, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/0546bf93956b4fc985caf081a01094cf, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/96e8711184954a1292ea3d21a1de1680, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/d7761e85977443c199683254340e3e97, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/f944e0425d2a4498a9565686f18d89e3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/65a27c567ea143dfa21175f9f778a326, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/39992cdc15c64d95a208a96454caa81b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/b5d9022a27174189a6a79b3e00030584, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/6025cb8ac0e344f5b61a9efffdde67fd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/efbbd76d221c41d59f72d4b702d43918, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/7dec2cc13d894b82aabcf05bcaff6d97, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/092e5b178c364295a3e472234519ec2b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/ee3837566a7c4b228d48aa0b9e8053f9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/907f6fd8177c4fe4b9f907050998eb46, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/50598927a4ef4286a7c2c184bc225c8e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/ce6cdc81379747b19eaa6490e7fe9646, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/9d0269a11ec247d296335fa24993aa92, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/37393577a2b84e94970ba6aa7f50adb8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/e22becfa7de248a4a8f35efe9b539456, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/5e0b624aff4f40edae242f4054eaabe4] to archive 2024-12-02T14:20:29,677 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:20:29,678 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/a3231e9cd78b43398b03349ed9795327 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/a3231e9cd78b43398b03349ed9795327 2024-12-02T14:20:29,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/c64b06c0db964977a2eecb128bf69a2b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/c64b06c0db964977a2eecb128bf69a2b 2024-12-02T14:20:29,680 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/aa63f39833d24ec4b09ab3f6e0521433 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/aa63f39833d24ec4b09ab3f6e0521433 2024-12-02T14:20:29,681 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/bbbd7619e0be4182bcb432d314f7bebe to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/bbbd7619e0be4182bcb432d314f7bebe 2024-12-02T14:20:29,682 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/e80c7a83f1f64d17a370e777c7979497 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/e80c7a83f1f64d17a370e777c7979497 2024-12-02T14:20:29,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/0546bf93956b4fc985caf081a01094cf to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/0546bf93956b4fc985caf081a01094cf 2024-12-02T14:20:29,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/96e8711184954a1292ea3d21a1de1680 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/96e8711184954a1292ea3d21a1de1680 2024-12-02T14:20:29,685 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/d7761e85977443c199683254340e3e97 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/d7761e85977443c199683254340e3e97 2024-12-02T14:20:29,686 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/f944e0425d2a4498a9565686f18d89e3 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/f944e0425d2a4498a9565686f18d89e3 2024-12-02T14:20:29,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/65a27c567ea143dfa21175f9f778a326 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/65a27c567ea143dfa21175f9f778a326 2024-12-02T14:20:29,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/39992cdc15c64d95a208a96454caa81b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/39992cdc15c64d95a208a96454caa81b 2024-12-02T14:20:29,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/b5d9022a27174189a6a79b3e00030584 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/b5d9022a27174189a6a79b3e00030584 2024-12-02T14:20:29,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/6025cb8ac0e344f5b61a9efffdde67fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/6025cb8ac0e344f5b61a9efffdde67fd 2024-12-02T14:20:29,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/efbbd76d221c41d59f72d4b702d43918 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/efbbd76d221c41d59f72d4b702d43918 2024-12-02T14:20:29,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/7dec2cc13d894b82aabcf05bcaff6d97 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/7dec2cc13d894b82aabcf05bcaff6d97 2024-12-02T14:20:29,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/092e5b178c364295a3e472234519ec2b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/092e5b178c364295a3e472234519ec2b 2024-12-02T14:20:29,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/ee3837566a7c4b228d48aa0b9e8053f9 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/ee3837566a7c4b228d48aa0b9e8053f9 2024-12-02T14:20:29,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/907f6fd8177c4fe4b9f907050998eb46 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/907f6fd8177c4fe4b9f907050998eb46 2024-12-02T14:20:29,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/50598927a4ef4286a7c2c184bc225c8e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/50598927a4ef4286a7c2c184bc225c8e 2024-12-02T14:20:29,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/ce6cdc81379747b19eaa6490e7fe9646 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/ce6cdc81379747b19eaa6490e7fe9646 2024-12-02T14:20:29,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/9d0269a11ec247d296335fa24993aa92 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/9d0269a11ec247d296335fa24993aa92 2024-12-02T14:20:29,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/37393577a2b84e94970ba6aa7f50adb8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/37393577a2b84e94970ba6aa7f50adb8 2024-12-02T14:20:29,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/e22becfa7de248a4a8f35efe9b539456 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/e22becfa7de248a4a8f35efe9b539456 2024-12-02T14:20:29,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/5e0b624aff4f40edae242f4054eaabe4 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/5e0b624aff4f40edae242f4054eaabe4 2024-12-02T14:20:29,705 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/recovered.edits/656.seqid, newMaxSeqId=656, maxSeqId=1 2024-12-02T14:20:29,706 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. 2024-12-02T14:20:29,706 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for c7cbc32e95f966c652e56aa6f1b3f54b: Waiting for close lock at 1733149228334Running coprocessor pre-close hooks at 1733149228334Disabling compacts and flushes for region at 1733149228334Disabling writes for close at 1733149228334Obtaining lock to block concurrent updates at 1733149228334Preparing flush snapshotting stores in c7cbc32e95f966c652e56aa6f1b3f54b at 1733149228334Finished memstore snapshotting TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733149228334Flushing stores of TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b. at 1733149228335 (+1 ms)Flushing c7cbc32e95f966c652e56aa6f1b3f54b/A: creating writer at 1733149228335Flushing c7cbc32e95f966c652e56aa6f1b3f54b/A: appending metadata at 1733149228337 (+2 ms)Flushing c7cbc32e95f966c652e56aa6f1b3f54b/A: closing flushed file at 1733149228337Flushing c7cbc32e95f966c652e56aa6f1b3f54b/B: creating writer at 1733149228758 (+421 ms)Flushing c7cbc32e95f966c652e56aa6f1b3f54b/B: appending metadata at 1733149228760 (+2 ms)Flushing c7cbc32e95f966c652e56aa6f1b3f54b/B: closing flushed file at 1733149228760Flushing c7cbc32e95f966c652e56aa6f1b3f54b/C: creating writer at 1733149229168 (+408 ms)Flushing c7cbc32e95f966c652e56aa6f1b3f54b/C: appending metadata at 1733149229170 (+2 ms)Flushing c7cbc32e95f966c652e56aa6f1b3f54b/C: closing flushed file at 1733149229170Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@560e00bf: reopening flushed file at 1733149229599 (+429 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b8c34d0: reopening flushed file at 1733149229605 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24f9f656: reopening flushed file at 1733149229610 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c7cbc32e95f966c652e56aa6f1b3f54b in 1284ms, sequenceid=653, compaction requested=true at 1733149229618 (+8 ms)Writing region close event to WAL at 1733149229703 (+85 ms)Running coprocessor post-close hooks at 1733149229706 (+3 ms)Closed at 1733149229706 2024-12-02T14:20:29,707 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:29,708 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=c7cbc32e95f966c652e56aa6f1b3f54b, regionState=CLOSED 2024-12-02T14:20:29,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:20:29,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-02T14:20:29,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure c7cbc32e95f966c652e56aa6f1b3f54b, server=a3a61c9ba14f,46793,1733149138543 in 1.5290 sec 2024-12-02T14:20:29,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-02T14:20:29,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7cbc32e95f966c652e56aa6f1b3f54b, UNASSIGN in 1.5340 sec 2024-12-02T14:20:29,716 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-02T14:20:29,716 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5380 sec 2024-12-02T14:20:29,718 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149229717"}]},"ts":"1733149229717"} 2024-12-02T14:20:29,719 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T14:20:29,719 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T14:20:29,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5490 sec 2024-12-02T14:20:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T14:20:30,319 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:30,320 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T14:20:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:30,322 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T14:20:30,323 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:30,324 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:30,326 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/recovered.edits] 2024-12-02T14:20:30,328 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/86d4fbb1863742de908aebd101975732 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/86d4fbb1863742de908aebd101975732 2024-12-02T14:20:30,330 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/9b02d160c68f4082891d75050b34c345 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/9b02d160c68f4082891d75050b34c345 2024-12-02T14:20:30,331 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/e4b7b2f8fb2f4ad586c2ed0b7c4555ef to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/A/e4b7b2f8fb2f4ad586c2ed0b7c4555ef 2024-12-02T14:20:30,333 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/0df906f256da46658b4045e47b1cd593 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/0df906f256da46658b4045e47b1cd593 2024-12-02T14:20:30,334 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/436bc90104f94336ac6367e5fa9b5d28 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/436bc90104f94336ac6367e5fa9b5d28 2024-12-02T14:20:30,335 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/56ec88e2f7e448388630f1d0c392ed1c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/B/56ec88e2f7e448388630f1d0c392ed1c 2024-12-02T14:20:30,337 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/1d6de0338ab24714a93fb78b50faa32e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/1d6de0338ab24714a93fb78b50faa32e 2024-12-02T14:20:30,338 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/283e2a5368754d70afdc5af356c01492 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/283e2a5368754d70afdc5af356c01492 2024-12-02T14:20:30,339 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/c5629d995aa441bd9632d61a16334021 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/C/c5629d995aa441bd9632d61a16334021 2024-12-02T14:20:30,343 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/recovered.edits/656.seqid to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b/recovered.edits/656.seqid 2024-12-02T14:20:30,343 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/c7cbc32e95f966c652e56aa6f1b3f54b 2024-12-02T14:20:30,343 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T14:20:30,345 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:30,349 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T14:20:30,351 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T14:20:30,352 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:30,352 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T14:20:30,353 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733149230352"}]},"ts":"9223372036854775807"} 2024-12-02T14:20:30,355 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T14:20:30,355 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c7cbc32e95f966c652e56aa6f1b3f54b, NAME => 'TestAcidGuarantees,,1733149203588.c7cbc32e95f966c652e56aa6f1b3f54b.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T14:20:30,355 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T14:20:30,355 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733149230355"}]},"ts":"9223372036854775807"} 2024-12-02T14:20:30,357 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T14:20:30,358 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:30,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-12-02T14:20:30,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T14:20:30,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T14:20:30,428 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:30,435 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-938368545_22 at /127.0.0.1:52720 [Waiting for operation #294] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_180533634_22 at /127.0.0.1:52886 [Waiting for operation #214] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=452 (was 378) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5842 (was 6084) 2024-12-02T14:20:30,445 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=452, ProcessCount=11, AvailableMemoryMB=5841 2024-12-02T14:20:30,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T14:20:30,447 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T14:20:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:30,449 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T14:20:30,449 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:30,449 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-12-02T14:20:30,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-02T14:20:30,450 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T14:20:30,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742150_1326 (size=1061) 2024-12-02T14:20:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-02T14:20:30,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-02T14:20:30,859 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919 2024-12-02T14:20:30,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742151_1327 (size=53) 2024-12-02T14:20:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-02T14:20:31,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:20:31,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 994773e29f6a48e9039005d3a6da70cd, disabling compactions & flushes 2024-12-02T14:20:31,266 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:31,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:31,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. after waiting 0 ms 2024-12-02T14:20:31,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:31,266 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:31,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 994773e29f6a48e9039005d3a6da70cd: Waiting for close lock at 1733149231266Disabling compacts and flushes for region at 1733149231266Disabling writes for close at 1733149231266Writing region close event to WAL at 1733149231266Closed at 1733149231266 2024-12-02T14:20:31,267 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T14:20:31,267 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733149231267"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733149231267"}]},"ts":"1733149231267"} 2024-12-02T14:20:31,269 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T14:20:31,269 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T14:20:31,269 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149231269"}]},"ts":"1733149231269"} 2024-12-02T14:20:31,271 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T14:20:31,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=994773e29f6a48e9039005d3a6da70cd, ASSIGN}] 2024-12-02T14:20:31,272 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=994773e29f6a48e9039005d3a6da70cd, ASSIGN 2024-12-02T14:20:31,272 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=994773e29f6a48e9039005d3a6da70cd, ASSIGN; state=OFFLINE, location=a3a61c9ba14f,46793,1733149138543; forceNewPlan=false, retain=false 2024-12-02T14:20:31,423 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=994773e29f6a48e9039005d3a6da70cd, regionState=OPENING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:31,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=994773e29f6a48e9039005d3a6da70cd, ASSIGN because future has completed 2024-12-02T14:20:31,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:20:31,585 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:31,585 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} 2024-12-02T14:20:31,586 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:31,586 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:20:31,586 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:31,587 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-02T14:20:31,589 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:31,591 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:20:31,591 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 994773e29f6a48e9039005d3a6da70cd columnFamilyName A 2024-12-02T14:20:31,591 DEBUG [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:31,592 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.HStore(327): Store=994773e29f6a48e9039005d3a6da70cd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:20:31,592 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:31,593 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:20:31,594 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 994773e29f6a48e9039005d3a6da70cd columnFamilyName B 2024-12-02T14:20:31,594 DEBUG [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:31,594 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.HStore(327): Store=994773e29f6a48e9039005d3a6da70cd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:20:31,594 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:31,596 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:20:31,596 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 994773e29f6a48e9039005d3a6da70cd columnFamilyName C 2024-12-02T14:20:31,596 DEBUG [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:31,596 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.HStore(327): Store=994773e29f6a48e9039005d3a6da70cd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:20:31,597 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:31,597 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:31,597 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:31,598 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:31,600 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:31,600 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:31,600 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T14:20:31,603 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:31,606 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T14:20:31,607 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 994773e29f6a48e9039005d3a6da70cd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60088196, jitterRate=-0.10461610555648804}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T14:20:31,607 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:31,608 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 994773e29f6a48e9039005d3a6da70cd: Running coprocessor pre-open hook at 1733149231587Writing region info on filesystem at 1733149231587Initializing all the Stores at 1733149231589 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149231589Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149231589Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149231589Cleaning up temporary data from old regions at 1733149231600 (+11 ms)Running coprocessor post-open hooks at 1733149231607 (+7 ms)Region opened successfully at 1733149231608 (+1 ms) 2024-12-02T14:20:31,609 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., pid=87, masterSystemTime=1733149231578 2024-12-02T14:20:31,612 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:31,612 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:31,613 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=994773e29f6a48e9039005d3a6da70cd, regionState=OPEN, openSeqNum=2, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:31,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:20:31,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-02T14:20:31,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 in 191 msec 2024-12-02T14:20:31,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-02T14:20:31,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=994773e29f6a48e9039005d3a6da70cd, ASSIGN in 347 msec 2024-12-02T14:20:31,620 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T14:20:31,620 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149231620"}]},"ts":"1733149231620"} 2024-12-02T14:20:31,621 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T14:20:31,622 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T14:20:31,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-12-02T14:20:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-02T14:20:32,600 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:32,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dda73f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:32,601 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:32,602 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:32,603 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7743c844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:32,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:32,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:32,607 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61899a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:32,608 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:32,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:32,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:32,611 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:32,612 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T14:20:32,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is a3a61c9ba14f,35877,1733149137941 2024-12-02T14:20:32,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@13323975 2024-12-02T14:20:32,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T14:20:32,614 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T14:20:32,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T14:20:32,615 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T14:20:32,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:32,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742152_1328 (size=1097) 2024-12-02T14:20:33,029 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T14:20:33,030 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T14:20:33,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T14:20:33,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=994773e29f6a48e9039005d3a6da70cd, REOPEN/MOVE}] 2024-12-02T14:20:33,035 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=994773e29f6a48e9039005d3a6da70cd, REOPEN/MOVE 2024-12-02T14:20:33,035 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=994773e29f6a48e9039005d3a6da70cd, regionState=CLOSING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:33,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=994773e29f6a48e9039005d3a6da70cd, REOPEN/MOVE because future has completed 2024-12-02T14:20:33,037 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T14:20:33,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:20:33,190 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:33,190 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T14:20:33,190 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 994773e29f6a48e9039005d3a6da70cd, disabling compactions & flushes 2024-12-02T14:20:33,190 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:33,190 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:33,190 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. after waiting 0 ms 2024-12-02T14:20:33,190 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:33,194 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T14:20:33,194 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:33,194 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 994773e29f6a48e9039005d3a6da70cd: Waiting for close lock at 1733149233190Running coprocessor pre-close hooks at 1733149233190Disabling compacts and flushes for region at 1733149233190Disabling writes for close at 1733149233190Writing region close event to WAL at 1733149233190Running coprocessor post-close hooks at 1733149233194 (+4 ms)Closed at 1733149233194 2024-12-02T14:20:33,194 WARN [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 994773e29f6a48e9039005d3a6da70cd to self. 2024-12-02T14:20:33,196 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:33,196 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=994773e29f6a48e9039005d3a6da70cd, regionState=CLOSED 2024-12-02T14:20:33,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:20:33,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-02T14:20:33,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 in 161 msec 2024-12-02T14:20:33,200 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=994773e29f6a48e9039005d3a6da70cd, REOPEN/MOVE; state=CLOSED, location=a3a61c9ba14f,46793,1733149138543; forceNewPlan=false, retain=true 2024-12-02T14:20:33,350 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=994773e29f6a48e9039005d3a6da70cd, regionState=OPENING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:33,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=994773e29f6a48e9039005d3a6da70cd, REOPEN/MOVE because future has completed 2024-12-02T14:20:33,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:20:33,507 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:33,507 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} 2024-12-02T14:20:33,507 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:33,507 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:20:33,507 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:33,507 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:33,508 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:33,509 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:20:33,509 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 994773e29f6a48e9039005d3a6da70cd columnFamilyName A 2024-12-02T14:20:33,510 DEBUG [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:33,511 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.HStore(327): Store=994773e29f6a48e9039005d3a6da70cd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:20:33,511 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:33,512 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:20:33,512 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 994773e29f6a48e9039005d3a6da70cd columnFamilyName B 2024-12-02T14:20:33,512 DEBUG [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:33,513 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.HStore(327): Store=994773e29f6a48e9039005d3a6da70cd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:20:33,513 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:33,513 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:20:33,514 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 994773e29f6a48e9039005d3a6da70cd columnFamilyName C 2024-12-02T14:20:33,514 DEBUG [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:33,514 INFO [StoreOpener-994773e29f6a48e9039005d3a6da70cd-1 {}] regionserver.HStore(327): Store=994773e29f6a48e9039005d3a6da70cd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:20:33,514 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:33,515 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:33,515 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:33,516 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:33,517 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:33,517 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:33,517 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T14:20:33,518 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:33,519 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 994773e29f6a48e9039005d3a6da70cd; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72923636, jitterRate=0.08664685487747192}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T14:20:33,519 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:33,519 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 994773e29f6a48e9039005d3a6da70cd: Running coprocessor pre-open hook at 1733149233507Writing region info on filesystem at 1733149233507Initializing all the Stores at 1733149233508 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149233508Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149233508Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149233508Cleaning up temporary data from old regions at 1733149233517 (+9 ms)Running coprocessor post-open hooks at 1733149233519 (+2 ms)Region opened successfully at 1733149233519 2024-12-02T14:20:33,520 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., pid=92, masterSystemTime=1733149233504 2024-12-02T14:20:33,522 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:33,522 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:33,522 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=994773e29f6a48e9039005d3a6da70cd, regionState=OPEN, openSeqNum=5, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:33,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:20:33,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-12-02T14:20:33,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 in 173 msec 2024-12-02T14:20:33,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-02T14:20:33,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=994773e29f6a48e9039005d3a6da70cd, REOPEN/MOVE in 493 msec 2024-12-02T14:20:33,534 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-02T14:20:33,534 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 499 msec 2024-12-02T14:20:33,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 919 msec 2024-12-02T14:20:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-02T14:20:33,538 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:33,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554bdeef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,538 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:33,539 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:33,539 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4882bb07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:33,541 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@399373d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5232d48c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,542 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:33,542 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:33,543 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:33,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b06950b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:33,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:33,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:33,545 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:33,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@751bffd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b67a6ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,546 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:33,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:33,547 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:33,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:33,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@545ea854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:33,549 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:33,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57958504, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12688331, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,550 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:33,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:33,551 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:33,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33650d0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:33,554 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:33,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27916a8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1456e4e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:33,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:33,556 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5557138c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:33,558 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:33,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d9c030, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74436e44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,559 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:33,559 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:33,560 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb54f95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:33,561 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:33,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@120120f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15099ff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,562 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:33,562 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:33,563 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a59f5b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:33,564 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69afbefd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbfebb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,566 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:33,566 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:33,567 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29027cac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:33,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:33,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:33,568 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:33,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d87cae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bdde2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:33,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:33,575 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:33,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:33,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52cc992b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:33,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:33,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:33,577 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:33,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eb3976, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7608485b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:20:33,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:33,581 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:20:33,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:33,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:20:33,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26220c9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:20:33,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:33,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:33,582 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:33,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26e37772, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:33,583 DEBUG [Thread-1506 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:33,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:33,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:33,584 DEBUG [Thread-1502 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:33,585 DEBUG [Thread-1511 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:33,585 DEBUG [Thread-1517 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:33,585 DEBUG [Thread-1515 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:33,585 DEBUG [Thread-1500 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:33,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:33,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:33,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:33,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:33,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:33,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:33,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:33,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:33,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:33,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:33,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:33,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:20:33,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:20:33,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:33,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:20:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:20:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:20:33,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:33,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:33,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:33,599 DEBUG [Thread-1513 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:20:33,602 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:20:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:33,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-02T14:20:33,605 DEBUG [Thread-1509 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:20:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:33,607 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-02T14:20:33,608 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:20:33,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:20:33,609 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:20:33,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:20:33,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T14:20:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:33,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:33,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:33,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:33,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:33,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:33,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:33,621 DEBUG [Thread-1498 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:33,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:33,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:33,624 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:20:33,633 DEBUG [Thread-1504 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:20:33,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:33,636 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:20:33,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024076a8b4f4544be5bf2ef45f09d729d7_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149233613/Put/seqid=0 2024-12-02T14:20:33,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149293655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:33,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149293661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:33,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149293665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:33,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149293669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:33,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149293673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:33,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:33,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742153_1329 (size=12154) 2024-12-02T14:20:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-02T14:20:33,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-02T14:20:33,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:33,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:33,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:33,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:33,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:33,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:33,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149293779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:33,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149293779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:33,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149293788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:33,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149293788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:33,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149293788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:33,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:33,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:33,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-02T14:20:33,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:33,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:33,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:33,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:33,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-02T14:20:33,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149293989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:33,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149293989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:34,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149293998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:34,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149293998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149293999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:34,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:34,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:34,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-02T14:20:34,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:34,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:34,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:34,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,107 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:34,118 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024076a8b4f4544be5bf2ef45f09d729d7_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024076a8b4f4544be5bf2ef45f09d729d7_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:34,121 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/eb08a389e003443daacc17f4d496c293, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:34,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/eb08a389e003443daacc17f4d496c293 is 175, key is test_row_0/A:col10/1733149233613/Put/seqid=0 2024-12-02T14:20:34,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742154_1330 (size=30955) 2024-12-02T14:20:34,139 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/eb08a389e003443daacc17f4d496c293 2024-12-02T14:20:34,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/991ad938bff142e6b995e950909f92d1 is 50, key is test_row_0/B:col10/1733149233613/Put/seqid=0 2024-12-02T14:20:34,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742155_1331 (size=12001) 2024-12-02T14:20:34,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-02T14:20:34,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:34,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:34,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:34,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-02T14:20:34,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:34,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149294299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:34,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:34,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149294308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:34,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:34,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149294308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:34,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:34,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149294318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:34,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149294318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:34,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:34,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-02T14:20:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:34,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:34,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-02T14:20:34,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:34,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:34,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:34,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/991ad938bff142e6b995e950909f92d1 2024-12-02T14:20:34,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/f7b0daa9c0894e89b43f7fbed9f570da is 50, key is test_row_0/C:col10/1733149233613/Put/seqid=0 2024-12-02T14:20:34,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742156_1332 (size=12001) 2024-12-02T14:20:34,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-02T14:20:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:34,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:34,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-02T14:20:34,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149294808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:34,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149294829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:34,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149294829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149294829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149294829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:34,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:34,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-02T14:20:34,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:34,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:34,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:34,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-02T14:20:34,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:34,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:34,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:34,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:34,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:35,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/f7b0daa9c0894e89b43f7fbed9f570da 2024-12-02T14:20:35,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/eb08a389e003443daacc17f4d496c293 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/eb08a389e003443daacc17f4d496c293 2024-12-02T14:20:35,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/eb08a389e003443daacc17f4d496c293, entries=150, sequenceid=16, filesize=30.2 K 2024-12-02T14:20:35,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/991ad938bff142e6b995e950909f92d1 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/991ad938bff142e6b995e950909f92d1 2024-12-02T14:20:35,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/991ad938bff142e6b995e950909f92d1, entries=150, sequenceid=16, filesize=11.7 K 2024-12-02T14:20:35,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/f7b0daa9c0894e89b43f7fbed9f570da as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/f7b0daa9c0894e89b43f7fbed9f570da 2024-12-02T14:20:35,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/f7b0daa9c0894e89b43f7fbed9f570da, entries=150, sequenceid=16, filesize=11.7 K 2024-12-02T14:20:35,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 994773e29f6a48e9039005d3a6da70cd in 1477ms, sequenceid=16, compaction requested=false 2024-12-02T14:20:35,090 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-02T14:20:35,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:35,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-02T14:20:35,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:35,152 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T14:20:35,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:35,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:35,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:35,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:35,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:35,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:35,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029389ba69621844e384cb01a7349581e4_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149233665/Put/seqid=0 2024-12-02T14:20:35,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742157_1333 (size=12154) 2024-12-02T14:20:35,229 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T14:20:35,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:35,629 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029389ba69621844e384cb01a7349581e4_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029389ba69621844e384cb01a7349581e4_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:35,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/ba89e28286504939b5d9ffd1911173ef, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:35,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/ba89e28286504939b5d9ffd1911173ef is 175, key is test_row_0/A:col10/1733149233665/Put/seqid=0 2024-12-02T14:20:35,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742158_1334 (size=30955) 2024-12-02T14:20:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-02T14:20:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:35,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:35,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149295850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:35,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149295850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:35,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149295859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:35,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:35,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149295862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149295862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:35,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:35,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149295969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149295969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:35,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149295978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:35,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:35,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149295978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149295978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:35,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,043 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/ba89e28286504939b5d9ffd1911173ef 2024-12-02T14:20:36,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/b064309d99074f529796b86c7fd88f12 is 50, key is test_row_0/B:col10/1733149233665/Put/seqid=0 2024-12-02T14:20:36,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742159_1335 (size=12001) 2024-12-02T14:20:36,065 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/b064309d99074f529796b86c7fd88f12 2024-12-02T14:20:36,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/4f79890517204c97844422134014a6c0 is 50, key is test_row_0/C:col10/1733149233665/Put/seqid=0 2024-12-02T14:20:36,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742160_1336 (size=12001) 2024-12-02T14:20:36,088 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/4f79890517204c97844422134014a6c0 2024-12-02T14:20:36,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/ba89e28286504939b5d9ffd1911173ef as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/ba89e28286504939b5d9ffd1911173ef 2024-12-02T14:20:36,100 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/ba89e28286504939b5d9ffd1911173ef, entries=150, sequenceid=36, filesize=30.2 K 2024-12-02T14:20:36,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/b064309d99074f529796b86c7fd88f12 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/b064309d99074f529796b86c7fd88f12 2024-12-02T14:20:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,105 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/b064309d99074f529796b86c7fd88f12, entries=150, sequenceid=36, filesize=11.7 K 2024-12-02T14:20:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/4f79890517204c97844422134014a6c0 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/4f79890517204c97844422134014a6c0 2024-12-02T14:20:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,109 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/4f79890517204c97844422134014a6c0, entries=150, sequenceid=36, filesize=11.7 K 2024-12-02T14:20:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,110 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 994773e29f6a48e9039005d3a6da70cd in 957ms, sequenceid=36, compaction requested=false 2024-12-02T14:20:36,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:36,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-02T14:20:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-02T14:20:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-02T14:20:36,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5020 sec 2024-12-02T14:20:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.5110 sec 2024-12-02T14:20:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:36,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T14:20:36,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:36,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:36,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:36,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:36,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:36,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:36,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412023f43476f0d7845ca9c6cb8c62c5e3b43_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149235858/Put/seqid=0 2024-12-02T14:20:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742161_1337 (size=12154) 2024-12-02T14:20:36,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149296287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149296288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149296290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149296291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149296292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149296408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149296409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149296409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149296409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149296409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,634 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:36,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149296629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149296629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149296629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149296629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149296629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,638 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412023f43476f0d7845ca9c6cb8c62c5e3b43_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412023f43476f0d7845ca9c6cb8c62c5e3b43_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:36,639 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/70349863ec654619900bf5d86bb22608, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:36,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/70349863ec654619900bf5d86bb22608 is 175, key is test_row_0/A:col10/1733149235858/Put/seqid=0 2024-12-02T14:20:36,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742162_1338 (size=30951) 2024-12-02T14:20:36,649 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/70349863ec654619900bf5d86bb22608 2024-12-02T14:20:36,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/aaefbe4272c94c15a90fa613c429cd31 is 50, key is test_row_0/B:col10/1733149235858/Put/seqid=0 2024-12-02T14:20:36,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742163_1339 (size=9657) 2024-12-02T14:20:36,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/aaefbe4272c94c15a90fa613c429cd31 2024-12-02T14:20:36,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/45231d57e47f40f2936c6a71265ed9b4 is 50, key is test_row_0/C:col10/1733149235858/Put/seqid=0 2024-12-02T14:20:36,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742164_1340 (size=9657) 2024-12-02T14:20:36,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149296939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149296939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149296939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149296948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:36,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:36,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149296948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:36,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:36,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:37,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/45231d57e47f40f2936c6a71265ed9b4 2024-12-02T14:20:37,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/70349863ec654619900bf5d86bb22608 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/70349863ec654619900bf5d86bb22608 2024-12-02T14:20:37,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/70349863ec654619900bf5d86bb22608, entries=150, sequenceid=49, filesize=30.2 K 2024-12-02T14:20:37,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/aaefbe4272c94c15a90fa613c429cd31 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/aaefbe4272c94c15a90fa613c429cd31 2024-12-02T14:20:37,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/aaefbe4272c94c15a90fa613c429cd31, entries=100, sequenceid=49, filesize=9.4 K 2024-12-02T14:20:37,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/45231d57e47f40f2936c6a71265ed9b4 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/45231d57e47f40f2936c6a71265ed9b4 2024-12-02T14:20:37,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/45231d57e47f40f2936c6a71265ed9b4, entries=100, sequenceid=49, filesize=9.4 K 2024-12-02T14:20:37,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 994773e29f6a48e9039005d3a6da70cd in 912ms, sequenceid=49, compaction requested=true 2024-12-02T14:20:37,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:37,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:20:37,126 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:37,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:37,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:20:37,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:37,126 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:37,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:20:37,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:37,129 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92861 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:37,129 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:37,130 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/A is initiating minor compaction (all files) 2024-12-02T14:20:37,130 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/B is initiating minor compaction (all files) 2024-12-02T14:20:37,130 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/A in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:37,130 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/B in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:37,130 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/eb08a389e003443daacc17f4d496c293, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/ba89e28286504939b5d9ffd1911173ef, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/70349863ec654619900bf5d86bb22608] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=90.7 K 2024-12-02T14:20:37,130 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/991ad938bff142e6b995e950909f92d1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/b064309d99074f529796b86c7fd88f12, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/aaefbe4272c94c15a90fa613c429cd31] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=32.9 K 2024-12-02T14:20:37,130 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:37,130 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/eb08a389e003443daacc17f4d496c293, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/ba89e28286504939b5d9ffd1911173ef, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/70349863ec654619900bf5d86bb22608] 2024-12-02T14:20:37,130 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 991ad938bff142e6b995e950909f92d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733149233607 2024-12-02T14:20:37,130 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb08a389e003443daacc17f4d496c293, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733149233607 2024-12-02T14:20:37,131 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting b064309d99074f529796b86c7fd88f12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733149233648 2024-12-02T14:20:37,131 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba89e28286504939b5d9ffd1911173ef, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733149233648 2024-12-02T14:20:37,131 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting aaefbe4272c94c15a90fa613c429cd31, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733149235858 2024-12-02T14:20:37,131 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70349863ec654619900bf5d86bb22608, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733149235858 2024-12-02T14:20:37,148 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#B#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:37,149 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/76e6cce1f598427888b876625bdb83c9 is 50, key is test_row_0/B:col10/1733149235858/Put/seqid=0 2024-12-02T14:20:37,152 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:37,174 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202b60a79edcd134467b0866fb317b87438_994773e29f6a48e9039005d3a6da70cd store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:37,175 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202b60a79edcd134467b0866fb317b87438_994773e29f6a48e9039005d3a6da70cd, store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:37,176 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b60a79edcd134467b0866fb317b87438_994773e29f6a48e9039005d3a6da70cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:37,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742166_1342 (size=4469) 2024-12-02T14:20:37,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742165_1341 (size=12104) 2024-12-02T14:20:37,206 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#A#compaction#289 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:37,207 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/5fc3d4d169a047e8a81cb376139961f3 is 175, key is test_row_0/A:col10/1733149235858/Put/seqid=0 2024-12-02T14:20:37,213 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/76e6cce1f598427888b876625bdb83c9 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/76e6cce1f598427888b876625bdb83c9 2024-12-02T14:20:37,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742167_1343 (size=31165) 2024-12-02T14:20:37,219 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/B of 994773e29f6a48e9039005d3a6da70cd into 76e6cce1f598427888b876625bdb83c9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:37,219 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:37,219 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/B, priority=13, startTime=1733149237126; duration=0sec 2024-12-02T14:20:37,219 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:37,219 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:B 2024-12-02T14:20:37,219 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:37,220 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:37,220 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/C is initiating minor compaction (all files) 2024-12-02T14:20:37,220 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/C in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:37,220 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/f7b0daa9c0894e89b43f7fbed9f570da, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/4f79890517204c97844422134014a6c0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/45231d57e47f40f2936c6a71265ed9b4] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=32.9 K 2024-12-02T14:20:37,220 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting f7b0daa9c0894e89b43f7fbed9f570da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733149233607 2024-12-02T14:20:37,221 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f79890517204c97844422134014a6c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733149233648 2024-12-02T14:20:37,221 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 45231d57e47f40f2936c6a71265ed9b4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733149235858 2024-12-02T14:20:37,230 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#C#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:37,232 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/ec5deed2171d47a8b28b906bfbe14dbe is 50, key is test_row_0/C:col10/1733149235858/Put/seqid=0 2024-12-02T14:20:37,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742168_1344 (size=12104) 2024-12-02T14:20:37,268 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/ec5deed2171d47a8b28b906bfbe14dbe as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ec5deed2171d47a8b28b906bfbe14dbe 2024-12-02T14:20:37,273 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/C of 994773e29f6a48e9039005d3a6da70cd into ec5deed2171d47a8b28b906bfbe14dbe(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:37,273 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:37,273 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/C, priority=13, startTime=1733149237126; duration=0sec 2024-12-02T14:20:37,273 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:37,273 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:C 2024-12-02T14:20:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:37,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T14:20:37,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:37,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:37,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:37,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:37,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:37,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:37,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027fd6f1481fcf446ea41756740ca80e7e_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149237460/Put/seqid=0 2024-12-02T14:20:37,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149297478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:37,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149297479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:37,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:37,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149297481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:37,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:37,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149297493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:37,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742169_1345 (size=14594) 2024-12-02T14:20:37,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149297493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:37,502 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:37,505 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027fd6f1481fcf446ea41756740ca80e7e_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027fd6f1481fcf446ea41756740ca80e7e_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:37,506 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/660db7534288460795658cc53b56f49a, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:37,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/660db7534288460795658cc53b56f49a is 175, key is test_row_0/A:col10/1733149237460/Put/seqid=0 2024-12-02T14:20:37,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742170_1346 (size=39549) 2024-12-02T14:20:37,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149297599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:37,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149297599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149297599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:37,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149297608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:37,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149297608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:37,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:37,623 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/5fc3d4d169a047e8a81cb376139961f3 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/5fc3d4d169a047e8a81cb376139961f3 2024-12-02T14:20:37,632 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/A of 994773e29f6a48e9039005d3a6da70cd into 5fc3d4d169a047e8a81cb376139961f3(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:37,632 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:37,632 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/A, priority=13, startTime=1733149237126; duration=0sec 2024-12-02T14:20:37,632 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:37,632 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:A 2024-12-02T14:20:37,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-02T14:20:37,769 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:37,770 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:20:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-02T14:20:37,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T14:20:37,774 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:20:37,774 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:20:37,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:20:37,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149297818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:37,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:37,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149297818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149297819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149297819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:37,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149297828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T14:20:37,915 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=71, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/660db7534288460795658cc53b56f49a 2024-12-02T14:20:37,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/8e2bce1d11d8418ebefb8b14caf30398 is 50, key is test_row_0/B:col10/1733149237460/Put/seqid=0 2024-12-02T14:20:37,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742171_1347 (size=12001) 2024-12-02T14:20:37,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T14:20:37,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:37,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:37,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:37,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:37,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:38,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T14:20:38,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:38,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:38,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:38,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:38,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T14:20:38,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149298128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:38,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:38,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149298128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:38,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:38,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149298129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:38,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:38,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149298129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:38,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149298138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:38,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T14:20:38,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:38,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:38,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:38,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:38,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:38,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:38,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/8e2bce1d11d8418ebefb8b14caf30398 2024-12-02T14:20:38,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/c93a7ecfcfd142a4b5648bb08668059e is 50, key is test_row_0/C:col10/1733149237460/Put/seqid=0 2024-12-02T14:20:38,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742172_1348 (size=12001) 2024-12-02T14:20:38,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T14:20:38,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:38,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:38,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:38,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:38,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T14:20:38,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T14:20:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:38,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:38,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149298638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:38,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149298639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:38,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149298649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:38,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149298649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:38,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149298658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:38,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T14:20:38,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:38,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:38,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:38,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:38,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:38,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/c93a7ecfcfd142a4b5648bb08668059e 2024-12-02T14:20:38,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/660db7534288460795658cc53b56f49a as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/660db7534288460795658cc53b56f49a 2024-12-02T14:20:38,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/660db7534288460795658cc53b56f49a, entries=200, sequenceid=71, filesize=38.6 K 2024-12-02T14:20:38,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/8e2bce1d11d8418ebefb8b14caf30398 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/8e2bce1d11d8418ebefb8b14caf30398 2024-12-02T14:20:38,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/8e2bce1d11d8418ebefb8b14caf30398, entries=150, sequenceid=71, filesize=11.7 K 2024-12-02T14:20:38,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/c93a7ecfcfd142a4b5648bb08668059e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/c93a7ecfcfd142a4b5648bb08668059e 2024-12-02T14:20:38,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/c93a7ecfcfd142a4b5648bb08668059e, entries=150, sequenceid=71, filesize=11.7 K 2024-12-02T14:20:38,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 994773e29f6a48e9039005d3a6da70cd in 1316ms, sequenceid=71, compaction requested=false 2024-12-02T14:20:38,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:38,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-02T14:20:38,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:38,853 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T14:20:38,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:38,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:38,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:38,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:38,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:38,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:38,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f50d8a0ea5fa4b62940368bcb8744e76_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149237487/Put/seqid=0 2024-12-02T14:20:38,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742173_1349 (size=12154) 2024-12-02T14:20:38,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T14:20:39,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:39,289 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f50d8a0ea5fa4b62940368bcb8744e76_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f50d8a0ea5fa4b62940368bcb8744e76_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:39,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/f9281e00e19349b9bbcf8e6ad4f4733f, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:39,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/f9281e00e19349b9bbcf8e6ad4f4733f is 175, key is test_row_0/A:col10/1733149237487/Put/seqid=0 2024-12-02T14:20:39,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742174_1350 (size=30955) 2024-12-02T14:20:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:39,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:39,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149299680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:39,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733149239675, max timestamp=1733149239690Num uniques 859; 2024-12-02T14:20:39,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T14:20:39,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733149239675, max timestamp=1733149239690Num uniques 859; 2024-12-02T14:20:39,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733149239675, max timestamp=1733149239690Num uniques 859; 2024-12-02T14:20:39,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T14:20:39,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T14:20:39,703 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/f9281e00e19349b9bbcf8e6ad4f4733f 2024-12-02T14:20:39,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/b5b2476116c04b6fa8c52a9ad6880683 is 50, key is test_row_0/B:col10/1733149237487/Put/seqid=0 2024-12-02T14:20:39,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742175_1351 (size=12001) 2024-12-02T14:20:39,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149299719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:39,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149299729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:39,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149299729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:39,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:39,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149299729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:39,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:39,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149299799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:39,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149299838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:39,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:39,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149299848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:39,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149299848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:39,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:39,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149299849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T14:20:40,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149300009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:40,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149300048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:40,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:40,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149300058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149300058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:40,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149300058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:40,120 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/b5b2476116c04b6fa8c52a9ad6880683 2024-12-02T14:20:40,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/7f293374c4b34d068ac1a08a5e217298 is 50, key is test_row_0/C:col10/1733149237487/Put/seqid=0 2024-12-02T14:20:40,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742176_1352 (size=12001) 2024-12-02T14:20:40,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149300329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:40,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:40,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149300368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:40,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:40,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149300368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:40,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:40,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:40,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149300378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:40,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:40,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149300378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:40,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:40,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:40,529 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/7f293374c4b34d068ac1a08a5e217298 2024-12-02T14:20:40,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/f9281e00e19349b9bbcf8e6ad4f4733f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/f9281e00e19349b9bbcf8e6ad4f4733f 2024-12-02T14:20:40,540 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/f9281e00e19349b9bbcf8e6ad4f4733f, entries=150, sequenceid=84, filesize=30.2 K 2024-12-02T14:20:40,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/b5b2476116c04b6fa8c52a9ad6880683 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/b5b2476116c04b6fa8c52a9ad6880683 2024-12-02T14:20:40,544 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/b5b2476116c04b6fa8c52a9ad6880683, entries=150, sequenceid=84, filesize=11.7 K 2024-12-02T14:20:40,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/7f293374c4b34d068ac1a08a5e217298 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7f293374c4b34d068ac1a08a5e217298 2024-12-02T14:20:40,549 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7f293374c4b34d068ac1a08a5e217298, entries=150, sequenceid=84, filesize=11.7 K 2024-12-02T14:20:40,550 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=134.18 KB/137400 for 994773e29f6a48e9039005d3a6da70cd in 1697ms, sequenceid=84, compaction requested=true 2024-12-02T14:20:40,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:40,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:40,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-02T14:20:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-02T14:20:40,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-02T14:20:40,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7780 sec 2024-12-02T14:20:40,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.7830 sec 2024-12-02T14:20:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:40,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-02T14:20:40,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:40,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:40,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:40,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:40,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:40,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:40,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202313ecb2e29634188bc7e7fbe1986a189_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149239728/Put/seqid=0 2024-12-02T14:20:40,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742177_1353 (size=14594) 2024-12-02T14:20:40,867 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:40,871 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202313ecb2e29634188bc7e7fbe1986a189_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202313ecb2e29634188bc7e7fbe1986a189_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:40,872 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/d771aa0294724b07a899d020d0174a2e, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:40,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/d771aa0294724b07a899d020d0174a2e is 175, key is test_row_0/A:col10/1733149239728/Put/seqid=0 2024-12-02T14:20:40,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742178_1354 (size=39549) 2024-12-02T14:20:40,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149300887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:40,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149300889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149300889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:40,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149300898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:40,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:40,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149300901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:40,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:40,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:41,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:41,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149301009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:41,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:41,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149301009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:41,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:41,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149301018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:41,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:41,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149301218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:41,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149301218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:41,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:41,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149301229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:41,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:41,276 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/d771aa0294724b07a899d020d0174a2e 2024-12-02T14:20:41,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/4f0033ab7ee94526b060991322eb475f is 50, key is test_row_0/B:col10/1733149239728/Put/seqid=0 2024-12-02T14:20:41,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742179_1355 (size=12001) 2024-12-02T14:20:41,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/4f0033ab7ee94526b060991322eb475f 2024-12-02T14:20:41,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/ce71ccd687d84228ab0050771f0c16e9 is 50, key is test_row_0/C:col10/1733149239728/Put/seqid=0 2024-12-02T14:20:41,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742180_1356 (size=12001) 2024-12-02T14:20:41,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/ce71ccd687d84228ab0050771f0c16e9 2024-12-02T14:20:41,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/d771aa0294724b07a899d020d0174a2e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/d771aa0294724b07a899d020d0174a2e 2024-12-02T14:20:41,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/d771aa0294724b07a899d020d0174a2e, entries=200, sequenceid=119, filesize=38.6 K 2024-12-02T14:20:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/4f0033ab7ee94526b060991322eb475f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/4f0033ab7ee94526b060991322eb475f 2024-12-02T14:20:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/4f0033ab7ee94526b060991322eb475f, entries=150, sequenceid=119, filesize=11.7 K 2024-12-02T14:20:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/ce71ccd687d84228ab0050771f0c16e9 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ce71ccd687d84228ab0050771f0c16e9 2024-12-02T14:20:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ce71ccd687d84228ab0050771f0c16e9, entries=150, sequenceid=119, filesize=11.7 K 2024-12-02T14:20:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for 994773e29f6a48e9039005d3a6da70cd in 476ms, sequenceid=119, compaction requested=true 2024-12-02T14:20:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:20:41,332 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:20:41,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:41,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:20:41,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:41,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:20:41,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T14:20:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,332 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:20:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,333 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141218 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:20:41,333 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:20:41,334 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/A is initiating minor compaction (all files) 2024-12-02T14:20:41,334 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/B is initiating minor compaction (all files) 2024-12-02T14:20:41,334 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/A in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:41,334 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/B in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,334 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/5fc3d4d169a047e8a81cb376139961f3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/660db7534288460795658cc53b56f49a, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/f9281e00e19349b9bbcf8e6ad4f4733f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/d771aa0294724b07a899d020d0174a2e] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=137.9 K 2024-12-02T14:20:41,334 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/76e6cce1f598427888b876625bdb83c9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/8e2bce1d11d8418ebefb8b14caf30398, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/b5b2476116c04b6fa8c52a9ad6880683, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/4f0033ab7ee94526b060991322eb475f] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=47.0 K 2024-12-02T14:20:41,334 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:41,334 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/5fc3d4d169a047e8a81cb376139961f3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/660db7534288460795658cc53b56f49a, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/f9281e00e19349b9bbcf8e6ad4f4733f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/d771aa0294724b07a899d020d0174a2e] 2024-12-02T14:20:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,334 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 76e6cce1f598427888b876625bdb83c9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733149233649 2024-12-02T14:20:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,334 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fc3d4d169a047e8a81cb376139961f3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733149233649 2024-12-02T14:20:41,334 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e2bce1d11d8418ebefb8b14caf30398, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1733149236288 2024-12-02T14:20:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,335 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 660db7534288460795658cc53b56f49a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1733149236288 2024-12-02T14:20:41,335 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting b5b2476116c04b6fa8c52a9ad6880683, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733149237479 2024-12-02T14:20:41,335 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9281e00e19349b9bbcf8e6ad4f4733f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733149237479 2024-12-02T14:20:41,335 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f0033ab7ee94526b060991322eb475f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733149239713 2024-12-02T14:20:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,335 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting d771aa0294724b07a899d020d0174a2e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733149239713 2024-12-02T14:20:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,341 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,343 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#B#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,343 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/9fc34163a4d64715b213213c3602340e is 50, key is test_row_0/B:col10/1733149239728/Put/seqid=0 2024-12-02T14:20:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,345 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412029d81e4cc0c6941a788d1eac180f9b940_994773e29f6a48e9039005d3a6da70cd store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,347 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412029d81e4cc0c6941a788d1eac180f9b940_994773e29f6a48e9039005d3a6da70cd, store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,347 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029d81e4cc0c6941a788d1eac180f9b940_994773e29f6a48e9039005d3a6da70cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742181_1357 (size=12241) 2024-12-02T14:20:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742182_1358 (size=4469) 2024-12-02T14:20:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:41,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T14:20:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c95554e500aa45f28ce65af8c0c9de06_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149240899/Put/seqid=0 2024-12-02T14:20:41,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742183_1359 (size=12204) 2024-12-02T14:20:41,611 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,617 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c95554e500aa45f28ce65af8c0c9de06_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c95554e500aa45f28ce65af8c0c9de06_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:41,618 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/b870db388c4e48b7b44f26e15f6bbfe5, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:41,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/b870db388c4e48b7b44f26e15f6bbfe5 is 175, key is test_row_0/A:col10/1733149240899/Put/seqid=0 2024-12-02T14:20:41,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742184_1360 (size=31001) 2024-12-02T14:20:41,637 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/b870db388c4e48b7b44f26e15f6bbfe5 2024-12-02T14:20:41,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733149241597, max timestamp=1733149241644Num uniques 859; 2024-12-02T14:20:41,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733149241597, max timestamp=1733149241644Num uniques 859; 2024-12-02T14:20:41,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733149241597, max timestamp=1733149241644Num uniques 859; 2024-12-02T14:20:41,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T14:20:41,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T14:20:41,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T14:20:41,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/f8b4888a3d124514a68dfbaca1ebb490 is 50, key is test_row_0/B:col10/1733149240899/Put/seqid=0 2024-12-02T14:20:41,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742185_1361 (size=9707) 2024-12-02T14:20:41,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/f8b4888a3d124514a68dfbaca1ebb490 2024-12-02T14:20:41,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/a13cc4ab99e9405097326ac563ef7d0b is 50, key is test_row_0/C:col10/1733149240899/Put/seqid=0 2024-12-02T14:20:41,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742186_1362 (size=9707) 2024-12-02T14:20:41,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149301689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:41,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149301690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:41,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149301701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:41,755 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#A#compaction#301 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:41,756 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/563c1db2a09a42ff8833e0bfc7d459d1 is 175, key is test_row_0/A:col10/1733149239728/Put/seqid=0 2024-12-02T14:20:41,757 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/9fc34163a4d64715b213213c3602340e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9fc34163a4d64715b213213c3602340e 2024-12-02T14:20:41,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742187_1363 (size=31195) 2024-12-02T14:20:41,761 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/B of 994773e29f6a48e9039005d3a6da70cd into 9fc34163a4d64715b213213c3602340e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:41,761 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:41,761 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/B, priority=12, startTime=1733149241332; duration=0sec 2024-12-02T14:20:41,761 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:41,761 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:B 2024-12-02T14:20:41,761 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:20:41,764 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:20:41,764 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/C is initiating minor compaction (all files) 2024-12-02T14:20:41,764 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/C in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:41,764 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ec5deed2171d47a8b28b906bfbe14dbe, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/c93a7ecfcfd142a4b5648bb08668059e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7f293374c4b34d068ac1a08a5e217298, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ce71ccd687d84228ab0050771f0c16e9] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=47.0 K 2024-12-02T14:20:41,769 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting ec5deed2171d47a8b28b906bfbe14dbe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733149233649 2024-12-02T14:20:41,769 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/563c1db2a09a42ff8833e0bfc7d459d1 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/563c1db2a09a42ff8833e0bfc7d459d1 2024-12-02T14:20:41,770 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting c93a7ecfcfd142a4b5648bb08668059e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1733149236288 2024-12-02T14:20:41,770 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f293374c4b34d068ac1a08a5e217298, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733149237479 2024-12-02T14:20:41,771 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting ce71ccd687d84228ab0050771f0c16e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733149239713 2024-12-02T14:20:41,775 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/A of 994773e29f6a48e9039005d3a6da70cd into 563c1db2a09a42ff8833e0bfc7d459d1(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:41,775 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:41,775 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/A, priority=12, startTime=1733149241332; duration=0sec 2024-12-02T14:20:41,776 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:41,776 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:A 2024-12-02T14:20:41,782 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#C#compaction#305 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:41,782 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/099ae44d52f2454f9990c3b2f4f002b8 is 50, key is test_row_0/C:col10/1733149239728/Put/seqid=0 2024-12-02T14:20:41,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742188_1364 (size=12241) 2024-12-02T14:20:41,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149301809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149301809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:41,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149301818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:41,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:41,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149301908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:41,914 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 994773e29f6a48e9039005d3a6da70cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:41,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149301919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:41,927 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 994773e29f6a48e9039005d3a6da70cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:41,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:41,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T14:20:41,928 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:41,928 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:20:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-02T14:20:41,930 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:20:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T14:20:41,930 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:20:41,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:20:42,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149302028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T14:20:42,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149302028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:42,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:42,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149302039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:42,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:42,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/a13cc4ab99e9405097326ac563ef7d0b 2024-12-02T14:20:42,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/b870db388c4e48b7b44f26e15f6bbfe5 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/b870db388c4e48b7b44f26e15f6bbfe5 2024-12-02T14:20:42,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/b870db388c4e48b7b44f26e15f6bbfe5, entries=150, sequenceid=130, filesize=30.3 K 2024-12-02T14:20:42,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/f8b4888a3d124514a68dfbaca1ebb490 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/f8b4888a3d124514a68dfbaca1ebb490 2024-12-02T14:20:42,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-02T14:20:42,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:42,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:42,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:42,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:42,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:42,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/f8b4888a3d124514a68dfbaca1ebb490, entries=100, sequenceid=130, filesize=9.5 K 2024-12-02T14:20:42,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/a13cc4ab99e9405097326ac563ef7d0b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/a13cc4ab99e9405097326ac563ef7d0b 2024-12-02T14:20:42,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/a13cc4ab99e9405097326ac563ef7d0b, entries=100, sequenceid=130, filesize=9.5 K 2024-12-02T14:20:42,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 994773e29f6a48e9039005d3a6da70cd in 523ms, sequenceid=130, compaction requested=false 2024-12-02T14:20:42,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:42,196 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/099ae44d52f2454f9990c3b2f4f002b8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/099ae44d52f2454f9990c3b2f4f002b8 2024-12-02T14:20:42,200 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/C of 994773e29f6a48e9039005d3a6da70cd into 099ae44d52f2454f9990c3b2f4f002b8(size=12.0 K), total size for store is 21.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:42,200 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:42,200 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/C, priority=12, startTime=1733149241332; duration=0sec 2024-12-02T14:20:42,200 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:42,200 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:C 2024-12-02T14:20:42,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-02T14:20:42,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:42,237 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-02T14:20:42,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:42,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:42,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:42,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:42,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:42,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:42,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202057eccfec9464507bb07ddbeb3826ef1_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149241689/Put/seqid=0 2024-12-02T14:20:42,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742189_1365 (size=12304) 2024-12-02T14:20:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T14:20:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:42,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:42,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149302397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:42,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149302406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:42,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149302407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:42,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149302518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:42,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:42,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:42,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149302528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:42,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:42,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149302528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T14:20:42,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:42,658 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202057eccfec9464507bb07ddbeb3826ef1_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202057eccfec9464507bb07ddbeb3826ef1_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:42,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/b866d2e02f0f45ada7cb6e8bf79640ac, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:42,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/b866d2e02f0f45ada7cb6e8bf79640ac is 175, key is test_row_0/A:col10/1733149241689/Put/seqid=0 2024-12-02T14:20:42,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742190_1366 (size=31105) 2024-12-02T14:20:42,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149302728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:42,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:42,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149302749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:42,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149302749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:43,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149303048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:43,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149303059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:43,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:43,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149303059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:43,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:43,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T14:20:43,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:43,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:43,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:43,086 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/b866d2e02f0f45ada7cb6e8bf79640ac 2024-12-02T14:20:43,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/6ac502bd478544adb70e21a319795516 is 50, key is test_row_0/B:col10/1733149241689/Put/seqid=0 2024-12-02T14:20:43,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742191_1367 (size=12151) 2024-12-02T14:20:43,512 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/6ac502bd478544adb70e21a319795516 2024-12-02T14:20:43,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/c0d9558db7dc47618d7bbba201997103 is 50, key is test_row_0/C:col10/1733149241689/Put/seqid=0 2024-12-02T14:20:43,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742192_1368 (size=12151) 2024-12-02T14:20:43,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149303568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:43,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:43,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:43,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:43,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149303578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:43,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:43,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149303578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:43,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:43,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:43,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:43,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:43,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:43,924 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/c0d9558db7dc47618d7bbba201997103 2024-12-02T14:20:43,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/b866d2e02f0f45ada7cb6e8bf79640ac as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/b866d2e02f0f45ada7cb6e8bf79640ac 2024-12-02T14:20:43,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149303928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:43,935 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/b866d2e02f0f45ada7cb6e8bf79640ac, entries=150, sequenceid=167, filesize=30.4 K 2024-12-02T14:20:43,935 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 994773e29f6a48e9039005d3a6da70cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:43,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:43,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/6ac502bd478544adb70e21a319795516 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/6ac502bd478544adb70e21a319795516 2024-12-02T14:20:43,939 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/6ac502bd478544adb70e21a319795516, entries=150, sequenceid=167, filesize=11.9 K 2024-12-02T14:20:43,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/c0d9558db7dc47618d7bbba201997103 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/c0d9558db7dc47618d7bbba201997103 2024-12-02T14:20:43,943 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/c0d9558db7dc47618d7bbba201997103, entries=150, sequenceid=167, filesize=11.9 K 2024-12-02T14:20:43,944 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 994773e29f6a48e9039005d3a6da70cd in 1708ms, sequenceid=167, compaction requested=true 2024-12-02T14:20:43,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:43,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:43,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-02T14:20:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-02T14:20:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:43,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T14:20:43,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:43,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-02T14:20:43,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0150 sec 2024-12-02T14:20:43,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.0200 sec 2024-12-02T14:20:43,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202201eddd16df54c218175f0d78b2ecbd2_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149243944/Put/seqid=0 2024-12-02T14:20:43,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742193_1369 (size=14794) 2024-12-02T14:20:44,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149304037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:44,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:44,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:44,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T14:20:44,079 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:44,079 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:20:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-02T14:20:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T14:20:44,080 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:20:44,081 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:20:44,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:20:44,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149304148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T14:20:44,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T14:20:44,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:44,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:44,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:44,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,362 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149304359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:44,365 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202201eddd16df54c218175f0d78b2ecbd2_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202201eddd16df54c218175f0d78b2ecbd2_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:44,366 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/e4ca5bc3155347d58b13e850025d8990, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:44,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/e4ca5bc3155347d58b13e850025d8990 is 175, key is test_row_0/A:col10/1733149243944/Put/seqid=0 2024-12-02T14:20:44,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742194_1370 (size=39749) 2024-12-02T14:20:44,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T14:20:44,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:44,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:44,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:44,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T14:20:44,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T14:20:44,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:44,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:44,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:44,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149304590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:44,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149304590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:44,597 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 994773e29f6a48e9039005d3a6da70cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:44,597 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 994773e29f6a48e9039005d3a6da70cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:44,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149304599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:44,606 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 994773e29f6a48e9039005d3a6da70cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:44,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:44,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:44,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:44,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149304668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:44,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T14:20:44,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:44,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:44,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:44,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T14:20:44,780 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/e4ca5bc3155347d58b13e850025d8990 2024-12-02T14:20:44,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/6deafd897fd74839a20fe0cb3b535e95 is 50, key is test_row_0/B:col10/1733149243944/Put/seqid=0 2024-12-02T14:20:44,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742195_1371 (size=12151) 2024-12-02T14:20:44,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T14:20:44,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:44,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:44,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:44,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T14:20:44,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:44,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:44,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:44,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:45,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T14:20:45,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:45,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:45,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:45,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:45,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:45,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149305178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:45,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/6deafd897fd74839a20fe0cb3b535e95 2024-12-02T14:20:45,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/332f6dc209d64d7997f656d29dba78fa is 50, key is test_row_0/C:col10/1733149243944/Put/seqid=0 2024-12-02T14:20:45,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742196_1372 (size=12151) 2024-12-02T14:20:45,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/332f6dc209d64d7997f656d29dba78fa 2024-12-02T14:20:45,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/e4ca5bc3155347d58b13e850025d8990 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/e4ca5bc3155347d58b13e850025d8990 2024-12-02T14:20:45,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T14:20:45,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/e4ca5bc3155347d58b13e850025d8990, entries=200, sequenceid=179, filesize=38.8 K 2024-12-02T14:20:45,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/6deafd897fd74839a20fe0cb3b535e95 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/6deafd897fd74839a20fe0cb3b535e95 2024-12-02T14:20:45,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/6deafd897fd74839a20fe0cb3b535e95, entries=150, sequenceid=179, filesize=11.9 K 2024-12-02T14:20:45,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/332f6dc209d64d7997f656d29dba78fa as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/332f6dc209d64d7997f656d29dba78fa 2024-12-02T14:20:45,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/332f6dc209d64d7997f656d29dba78fa, entries=150, sequenceid=179, filesize=11.9 K 2024-12-02T14:20:45,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 994773e29f6a48e9039005d3a6da70cd in 1305ms, sequenceid=179, compaction requested=true 2024-12-02T14:20:45,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:45,250 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:20:45,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:20:45,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:45,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:20:45,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:45,250 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:20:45,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:20:45,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:45,252 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:20:45,252 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46250 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:20:45,252 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/A is initiating minor compaction (all files) 2024-12-02T14:20:45,252 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/B is initiating minor compaction (all files) 2024-12-02T14:20:45,252 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/A in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:45,252 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/B in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:45,252 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9fc34163a4d64715b213213c3602340e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/f8b4888a3d124514a68dfbaca1ebb490, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/6ac502bd478544adb70e21a319795516, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/6deafd897fd74839a20fe0cb3b535e95] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=45.2 K 2024-12-02T14:20:45,252 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/563c1db2a09a42ff8833e0bfc7d459d1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/b870db388c4e48b7b44f26e15f6bbfe5, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/b866d2e02f0f45ada7cb6e8bf79640ac, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/e4ca5bc3155347d58b13e850025d8990] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=129.9 K 2024-12-02T14:20:45,253 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:45,253 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/563c1db2a09a42ff8833e0bfc7d459d1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/b870db388c4e48b7b44f26e15f6bbfe5, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/b866d2e02f0f45ada7cb6e8bf79640ac, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/e4ca5bc3155347d58b13e850025d8990] 2024-12-02T14:20:45,253 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fc34163a4d64715b213213c3602340e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733149239713 2024-12-02T14:20:45,253 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting f8b4888a3d124514a68dfbaca1ebb490, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733149240899 2024-12-02T14:20:45,253 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 563c1db2a09a42ff8833e0bfc7d459d1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733149239713 2024-12-02T14:20:45,253 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting b870db388c4e48b7b44f26e15f6bbfe5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733149240887 2024-12-02T14:20:45,253 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ac502bd478544adb70e21a319795516, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733149241674 2024-12-02T14:20:45,254 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting b866d2e02f0f45ada7cb6e8bf79640ac, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733149241674 2024-12-02T14:20:45,254 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 6deafd897fd74839a20fe0cb3b535e95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733149242395 2024-12-02T14:20:45,254 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4ca5bc3155347d58b13e850025d8990, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733149242395 2024-12-02T14:20:45,269 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:45,271 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#B#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:45,271 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/d9f9210c1dfb4c039bec1a01733542ba is 50, key is test_row_0/B:col10/1733149243944/Put/seqid=0 2024-12-02T14:20:45,272 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412029500fa7f056e46c8b2a1a378e57c6e38_994773e29f6a48e9039005d3a6da70cd store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:45,274 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412029500fa7f056e46c8b2a1a378e57c6e38_994773e29f6a48e9039005d3a6da70cd, store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:45,275 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029500fa7f056e46c8b2a1a378e57c6e38_994773e29f6a48e9039005d3a6da70cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:45,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742197_1373 (size=12527) 2024-12-02T14:20:45,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742198_1374 (size=4469) 2024-12-02T14:20:45,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T14:20:45,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:45,302 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T14:20:45,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:45,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:45,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:45,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:45,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:45,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:45,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d0601d7041864ebe8a9137a4266602c5_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149243994/Put/seqid=0 2024-12-02T14:20:45,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742199_1375 (size=12304) 2024-12-02T14:20:45,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,315 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d0601d7041864ebe8a9137a4266602c5_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d0601d7041864ebe8a9137a4266602c5_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:45,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/aeacf69e605147a3a16a9ec86dc39f52, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:45,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/aeacf69e605147a3a16a9ec86dc39f52 is 175, key is test_row_0/A:col10/1733149243994/Put/seqid=0 2024-12-02T14:20:45,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742200_1376 (size=31105) 2024-12-02T14:20:45,686 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#A#compaction#313 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:45,686 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/a78912245cfb4a348b7805dea1548010 is 175, key is test_row_0/A:col10/1733149243944/Put/seqid=0 2024-12-02T14:20:45,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742201_1377 (size=31481) 2024-12-02T14:20:45,696 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/d9f9210c1dfb4c039bec1a01733542ba as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/d9f9210c1dfb4c039bec1a01733542ba 2024-12-02T14:20:45,704 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/a78912245cfb4a348b7805dea1548010 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/a78912245cfb4a348b7805dea1548010 2024-12-02T14:20:45,705 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/B of 994773e29f6a48e9039005d3a6da70cd into d9f9210c1dfb4c039bec1a01733542ba(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:45,705 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:45,705 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/B, priority=12, startTime=1733149245250; duration=0sec 2024-12-02T14:20:45,705 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:45,705 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:B 2024-12-02T14:20:45,705 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:20:45,707 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46250 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:20:45,707 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/C is initiating minor compaction (all files) 2024-12-02T14:20:45,707 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/C in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:45,707 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/099ae44d52f2454f9990c3b2f4f002b8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/a13cc4ab99e9405097326ac563ef7d0b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/c0d9558db7dc47618d7bbba201997103, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/332f6dc209d64d7997f656d29dba78fa] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=45.2 K 2024-12-02T14:20:45,708 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 099ae44d52f2454f9990c3b2f4f002b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733149239713 2024-12-02T14:20:45,708 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting a13cc4ab99e9405097326ac563ef7d0b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733149240899 2024-12-02T14:20:45,709 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/A of 994773e29f6a48e9039005d3a6da70cd into a78912245cfb4a348b7805dea1548010(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:45,709 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:45,709 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/A, priority=12, startTime=1733149245250; duration=0sec 2024-12-02T14:20:45,709 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:45,709 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:A 2024-12-02T14:20:45,710 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting c0d9558db7dc47618d7bbba201997103, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733149241674 2024-12-02T14:20:45,710 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 332f6dc209d64d7997f656d29dba78fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733149242395 2024-12-02T14:20:45,722 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/aeacf69e605147a3a16a9ec86dc39f52 2024-12-02T14:20:45,725 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#C#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:45,726 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/147536d3570d4796beee2fcedc33e1a8 is 50, key is test_row_0/C:col10/1733149243944/Put/seqid=0 2024-12-02T14:20:45,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/9646e64c938e44c39f8ac78fada582e9 is 50, key is test_row_0/B:col10/1733149243994/Put/seqid=0 2024-12-02T14:20:45,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742202_1378 (size=12527) 2024-12-02T14:20:45,755 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/147536d3570d4796beee2fcedc33e1a8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/147536d3570d4796beee2fcedc33e1a8 2024-12-02T14:20:45,761 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/C of 994773e29f6a48e9039005d3a6da70cd into 147536d3570d4796beee2fcedc33e1a8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:45,761 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:45,761 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/C, priority=12, startTime=1733149245250; duration=0sec 2024-12-02T14:20:45,761 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:45,761 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:C 2024-12-02T14:20:45,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742203_1379 (size=12151) 2024-12-02T14:20:45,776 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/9646e64c938e44c39f8ac78fada582e9 2024-12-02T14:20:45,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/7e6d9fe9d5c046c3a1feafacc183b164 is 50, key is test_row_0/C:col10/1733149243994/Put/seqid=0 2024-12-02T14:20:45,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742204_1380 (size=12151) 2024-12-02T14:20:45,839 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/7e6d9fe9d5c046c3a1feafacc183b164 2024-12-02T14:20:45,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/aeacf69e605147a3a16a9ec86dc39f52 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/aeacf69e605147a3a16a9ec86dc39f52 2024-12-02T14:20:45,851 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/aeacf69e605147a3a16a9ec86dc39f52, entries=150, sequenceid=199, filesize=30.4 K 2024-12-02T14:20:45,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/9646e64c938e44c39f8ac78fada582e9 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9646e64c938e44c39f8ac78fada582e9 2024-12-02T14:20:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,857 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9646e64c938e44c39f8ac78fada582e9, entries=150, sequenceid=199, filesize=11.9 K 2024-12-02T14:20:45,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/7e6d9fe9d5c046c3a1feafacc183b164 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7e6d9fe9d5c046c3a1feafacc183b164 2024-12-02T14:20:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,864 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7e6d9fe9d5c046c3a1feafacc183b164, entries=150, sequenceid=199, filesize=11.9 K 2024-12-02T14:20:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,865 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 994773e29f6a48e9039005d3a6da70cd in 563ms, sequenceid=199, compaction requested=false 2024-12-02T14:20:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:45,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:45,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-02T14:20:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-02T14:20:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-02T14:20:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7860 sec 2024-12-02T14:20:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.7930 sec 2024-12-02T14:20:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T14:20:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,229 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,229 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:20:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-02T14:20:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,231 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:20:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T14:20:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,232 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:20:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:20:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T14:20:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T14:20:46,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,385 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-02T14:20:46,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:46,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:46,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:46,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:46,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:46,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120223490b5c097640aa9773f7b73a7d72eb_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_1/A:col10/1733149246381/Put/seqid=0 2024-12-02T14:20:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742205_1381 (size=9814) 2024-12-02T14:20:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,401 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120223490b5c097640aa9773f7b73a7d72eb_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120223490b5c097640aa9773f7b73a7d72eb_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/67679d1f67024e909deead690e1492d9, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/67679d1f67024e909deead690e1492d9 is 175, key is test_row_1/A:col10/1733149246381/Put/seqid=0 2024-12-02T14:20:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742206_1382 (size=22461) 2024-12-02T14:20:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,417 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/67679d1f67024e909deead690e1492d9 2024-12-02T14:20:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/a34b5372acae4be28f94b62eaf440029 is 50, key is test_row_1/B:col10/1733149246381/Put/seqid=0 2024-12-02T14:20:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742207_1383 (size=9757) 2024-12-02T14:20:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,434 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/a34b5372acae4be28f94b62eaf440029 2024-12-02T14:20:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/463c78ddcd7d45b0a1cb4db1c896faca is 50, key is test_row_1/C:col10/1733149246381/Put/seqid=0 2024-12-02T14:20:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742208_1384 (size=9757) 2024-12-02T14:20:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:46,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T14:20:46,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149246588, max timestamp=1733149246605Num uniques 859; 2024-12-02T14:20:46,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149246588, max timestamp=1733149246605Num uniques 859; 2024-12-02T14:20:46,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:20:46,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:20:46,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149246588, max timestamp=1733149246605Num uniques 859; 2024-12-02T14:20:46,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:20:46,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733149246643, max timestamp=1733149246657Num uniques 1018; 2024-12-02T14:20:46,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-02T14:20:46,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733149246643, max timestamp=1733149246657Num uniques 1018; 2024-12-02T14:20:46,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-02T14:20:46,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733149246643, max timestamp=1733149246657Num uniques 1018; 2024-12-02T14:20:46,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-02T14:20:46,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733149246680, max timestamp=1733149246698Num uniques 1027; 2024-12-02T14:20:46,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733149246680, max timestamp=1733149246698Num uniques 1027; 2024-12-02T14:20:46,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733149246680, max timestamp=1733149246698Num uniques 1027; 2024-12-02T14:20:46,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-02T14:20:46,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-02T14:20:46,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-02T14:20:46,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733149246728, max timestamp=1733149246739Num uniques 1086; 2024-12-02T14:20:46,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733149246728, max timestamp=1733149246739Num uniques 1086; 2024-12-02T14:20:46,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733149246728, max timestamp=1733149246739Num uniques 1086; 2024-12-02T14:20:46,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-02T14:20:46,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-02T14:20:46,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-02T14:20:46,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733149246781, max timestamp=1733149246792Num uniques 1145; 2024-12-02T14:20:46,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-12-02T14:20:46,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733149246781, max timestamp=1733149246792Num uniques 1145; 2024-12-02T14:20:46,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-12-02T14:20:46,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733149246781, max timestamp=1733149246792Num uniques 1145; 2024-12-02T14:20:46,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-12-02T14:20:46,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733149246811, max timestamp=1733149246827Num uniques 1104; 2024-12-02T14:20:46,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-02T14:20:46,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733149246811, max timestamp=1733149246827Num uniques 1104; 2024-12-02T14:20:46,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-02T14:20:46,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733149246811, max timestamp=1733149246827Num uniques 1104; 2024-12-02T14:20:46,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-12-02T14:20:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T14:20:46,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1733149246849, max timestamp=1733149246866Num uniques 1063; 2024-12-02T14:20:46,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=14328, suffix heap size=138520,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=313 2024-12-02T14:20:46,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1733149246849, max timestamp=1733149246866Num uniques 1063; 2024-12-02T14:20:46,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=14328, suffix heap size=138520,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=313 2024-12-02T14:20:46,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733149246851, max timestamp=1733149246866Num uniques 1063; 2024-12-02T14:20:46,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=12038, suffix heap size=138520,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=263 2024-12-02T14:20:46,883 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/463c78ddcd7d45b0a1cb4db1c896faca 2024-12-02T14:20:46,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/67679d1f67024e909deead690e1492d9 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/67679d1f67024e909deead690e1492d9 2024-12-02T14:20:46,889 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/67679d1f67024e909deead690e1492d9, entries=100, sequenceid=208, filesize=21.9 K 2024-12-02T14:20:46,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/a34b5372acae4be28f94b62eaf440029 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/a34b5372acae4be28f94b62eaf440029 2024-12-02T14:20:46,894 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/a34b5372acae4be28f94b62eaf440029, entries=100, sequenceid=208, filesize=9.5 K 2024-12-02T14:20:46,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/463c78ddcd7d45b0a1cb4db1c896faca as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/463c78ddcd7d45b0a1cb4db1c896faca 2024-12-02T14:20:46,898 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/463c78ddcd7d45b0a1cb4db1c896faca, entries=100, sequenceid=208, filesize=9.5 K 2024-12-02T14:20:46,899 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=105.11 KB/107630 for 994773e29f6a48e9039005d3a6da70cd in 514ms, sequenceid=208, compaction requested=true 2024-12-02T14:20:46,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:46,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:46,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-02T14:20:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-02T14:20:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:46,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=111.82 KB heapSize=305.70 KB 2024-12-02T14:20:46,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:46,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:46,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:46,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:46,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:46,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:20:46,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-02T14:20:46,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 668 msec 2024-12-02T14:20:46,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 674 msec 2024-12-02T14:20:46,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120288678dd1c0fc491887d5f6a162d60c1e_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149246900/Put/seqid=0 2024-12-02T14:20:46,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742209_1385 (size=14994) 2024-12-02T14:20:46,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:46,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149306922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:46,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:46,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:46,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:46,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149306929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:46,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149306933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:46,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:46,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149306934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:46,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:47,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:47,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149307039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:47,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:47,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149307039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:47,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:47,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149307049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:47,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:47,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149307049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:47,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:47,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:47,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149307249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:47,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149307249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:47,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149307258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:47,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149307258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:47,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:47,327 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:47,330 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120288678dd1c0fc491887d5f6a162d60c1e_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120288678dd1c0fc491887d5f6a162d60c1e_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:47,331 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/38472b30296a49f58f67da16ed623f4f, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:47,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/38472b30296a49f58f67da16ed623f4f is 175, key is test_row_0/A:col10/1733149246900/Put/seqid=0 2024-12-02T14:20:47,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742210_1386 (size=39949) 2024-12-02T14:20:47,337 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/38472b30296a49f58f67da16ed623f4f 2024-12-02T14:20:47,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/395c627f0825436780d989098a655459 is 50, key is test_row_0/B:col10/1733149246900/Put/seqid=0 2024-12-02T14:20:47,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742211_1387 (size=12301) 2024-12-02T14:20:47,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/395c627f0825436780d989098a655459 2024-12-02T14:20:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T14:20:47,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/b461e22086ee4342ba30cafeee2c3ccd is 50, key is test_row_0/C:col10/1733149246900/Put/seqid=0 2024-12-02T14:20:47,371 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:47,372 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:20:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-02T14:20:47,374 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:20:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T14:20:47,374 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:20:47,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:20:47,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742212_1388 (size=12301) 2024-12-02T14:20:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T14:20:47,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T14:20:47,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:47,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:47,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:47,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:47,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:47,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149307558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:47,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149307558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:47,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149307568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:47,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149307568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:47,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T14:20:47,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:47,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:47,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:47,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:47,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T14:20:47,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/b461e22086ee4342ba30cafeee2c3ccd 2024-12-02T14:20:47,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/38472b30296a49f58f67da16ed623f4f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/38472b30296a49f58f67da16ed623f4f 2024-12-02T14:20:47,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/38472b30296a49f58f67da16ed623f4f, entries=200, sequenceid=342, filesize=39.0 K 2024-12-02T14:20:47,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/395c627f0825436780d989098a655459 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/395c627f0825436780d989098a655459 2024-12-02T14:20:47,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/395c627f0825436780d989098a655459, entries=150, sequenceid=342, filesize=12.0 K 2024-12-02T14:20:47,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/b461e22086ee4342ba30cafeee2c3ccd as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/b461e22086ee4342ba30cafeee2c3ccd 2024-12-02T14:20:47,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/b461e22086ee4342ba30cafeee2c3ccd, entries=150, sequenceid=342, filesize=12.0 K 2024-12-02T14:20:47,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~111.82 KB/114500, heapSize ~305.66 KB/312992, currentSize=73.80 KB/75570 for 994773e29f6a48e9039005d3a6da70cd in 895ms, sequenceid=342, compaction requested=true 2024-12-02T14:20:47,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:47,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:20:47,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:47,796 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:20:47,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:20:47,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:47,796 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:20:47,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:20:47,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:47,797 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46736 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:20:47,797 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124996 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:20:47,798 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/B is initiating minor compaction (all files) 2024-12-02T14:20:47,798 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/A is initiating minor compaction (all files) 2024-12-02T14:20:47,798 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/B in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:47,798 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/A in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:47,798 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/d9f9210c1dfb4c039bec1a01733542ba, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9646e64c938e44c39f8ac78fada582e9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/a34b5372acae4be28f94b62eaf440029, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/395c627f0825436780d989098a655459] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=45.6 K 2024-12-02T14:20:47,798 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/a78912245cfb4a348b7805dea1548010, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/aeacf69e605147a3a16a9ec86dc39f52, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/67679d1f67024e909deead690e1492d9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/38472b30296a49f58f67da16ed623f4f] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=122.1 K 2024-12-02T14:20:47,798 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:47,798 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/a78912245cfb4a348b7805dea1548010, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/aeacf69e605147a3a16a9ec86dc39f52, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/67679d1f67024e909deead690e1492d9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/38472b30296a49f58f67da16ed623f4f] 2024-12-02T14:20:47,798 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting d9f9210c1dfb4c039bec1a01733542ba, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733149242395 2024-12-02T14:20:47,798 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting a78912245cfb4a348b7805dea1548010, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733149242395 2024-12-02T14:20:47,798 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 9646e64c938e44c39f8ac78fada582e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733149243994 2024-12-02T14:20:47,799 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeacf69e605147a3a16a9ec86dc39f52, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733149243994 2024-12-02T14:20:47,799 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting a34b5372acae4be28f94b62eaf440029, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733149246318 2024-12-02T14:20:47,799 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67679d1f67024e909deead690e1492d9, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733149246318 2024-12-02T14:20:47,799 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 395c627f0825436780d989098a655459, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733149246889 2024-12-02T14:20:47,799 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38472b30296a49f58f67da16ed623f4f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733149246889 2024-12-02T14:20:47,805 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:47,805 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/5069ae3689ef4a7ba18bda3f333d5bb1 is 50, key is test_row_0/B:col10/1733149246900/Put/seqid=0 2024-12-02T14:20:47,806 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:47,808 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120207f72aa9e5c949f5853b655e446b2e8f_994773e29f6a48e9039005d3a6da70cd store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:47,811 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120207f72aa9e5c949f5853b655e446b2e8f_994773e29f6a48e9039005d3a6da70cd, store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:47,811 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120207f72aa9e5c949f5853b655e446b2e8f_994773e29f6a48e9039005d3a6da70cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:47,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742213_1389 (size=12813) 2024-12-02T14:20:47,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742214_1390 (size=4469) 2024-12-02T14:20:47,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T14:20:47,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:47,832 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T14:20:47,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:47,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:47,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:47,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:47,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:47,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:47,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bb46a6f5d3a54be5a36f5d8bf0af1b6f_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149246927/Put/seqid=0 2024-12-02T14:20:47,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742215_1391 (size=12454) 2024-12-02T14:20:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:47,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T14:20:48,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:48,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149308079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:48,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149308079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:48,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149308080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:48,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149308088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:48,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:48,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149308088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:48,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:48,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:48,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:48,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:48,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149308198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:48,215 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/5069ae3689ef4a7ba18bda3f333d5bb1 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/5069ae3689ef4a7ba18bda3f333d5bb1 2024-12-02T14:20:48,219 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#A#compaction#325 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:48,220 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/0d71dc895b3f47ed96ad92f6818efad7 is 175, key is test_row_0/A:col10/1733149246900/Put/seqid=0 2024-12-02T14:20:48,220 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/B of 994773e29f6a48e9039005d3a6da70cd into 5069ae3689ef4a7ba18bda3f333d5bb1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:48,220 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:48,220 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/B, priority=12, startTime=1733149247796; duration=0sec 2024-12-02T14:20:48,220 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:48,220 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:B 2024-12-02T14:20:48,220 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:20:48,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742216_1392 (size=31767) 2024-12-02T14:20:48,223 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46736 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:20:48,223 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/C is initiating minor compaction (all files) 2024-12-02T14:20:48,223 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/C in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:48,224 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/147536d3570d4796beee2fcedc33e1a8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7e6d9fe9d5c046c3a1feafacc183b164, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/463c78ddcd7d45b0a1cb4db1c896faca, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/b461e22086ee4342ba30cafeee2c3ccd] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=45.6 K 2024-12-02T14:20:48,224 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 147536d3570d4796beee2fcedc33e1a8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733149242395 2024-12-02T14:20:48,224 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e6d9fe9d5c046c3a1feafacc183b164, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733149243994 2024-12-02T14:20:48,226 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 463c78ddcd7d45b0a1cb4db1c896faca, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733149246318 2024-12-02T14:20:48,227 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting b461e22086ee4342ba30cafeee2c3ccd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733149246889 2024-12-02T14:20:48,227 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/0d71dc895b3f47ed96ad92f6818efad7 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/0d71dc895b3f47ed96ad92f6818efad7 2024-12-02T14:20:48,231 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/A of 994773e29f6a48e9039005d3a6da70cd into 0d71dc895b3f47ed96ad92f6818efad7(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:48,231 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:48,231 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/A, priority=12, startTime=1733149247796; duration=0sec 2024-12-02T14:20:48,231 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:48,231 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:A 2024-12-02T14:20:48,234 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#C#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:48,234 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/b83c57aedf2742aba90eaba3bd811e94 is 50, key is test_row_0/C:col10/1733149246900/Put/seqid=0 2024-12-02T14:20:48,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742217_1393 (size=12813) 2024-12-02T14:20:48,246 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/b83c57aedf2742aba90eaba3bd811e94 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/b83c57aedf2742aba90eaba3bd811e94 2024-12-02T14:20:48,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:48,254 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/C of 994773e29f6a48e9039005d3a6da70cd into b83c57aedf2742aba90eaba3bd811e94(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:48,254 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:48,254 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/C, priority=12, startTime=1733149247796; duration=0sec 2024-12-02T14:20:48,255 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:48,255 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:C 2024-12-02T14:20:48,257 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bb46a6f5d3a54be5a36f5d8bf0af1b6f_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bb46a6f5d3a54be5a36f5d8bf0af1b6f_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:48,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/5242addf753a4f2e86ed4325f5e428be, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:48,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/5242addf753a4f2e86ed4325f5e428be is 175, key is test_row_0/A:col10/1733149246927/Put/seqid=0 2024-12-02T14:20:48,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742218_1394 (size=31255) 2024-12-02T14:20:48,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149308408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T14:20:48,665 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/5242addf753a4f2e86ed4325f5e428be 2024-12-02T14:20:48,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/4ea0f2ee2be54469b906ae7e23338c8f is 50, key is test_row_0/B:col10/1733149246927/Put/seqid=0 2024-12-02T14:20:48,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742219_1395 (size=12301) 2024-12-02T14:20:48,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149308718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:49,082 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/4ea0f2ee2be54469b906ae7e23338c8f 2024-12-02T14:20:49,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/7cc9b2c6ebcb4c94a48c2935a34ee258 is 50, key is test_row_0/C:col10/1733149246927/Put/seqid=0 2024-12-02T14:20:49,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742220_1396 (size=12301) 2024-12-02T14:20:49,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149309098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:49,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149309098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:49,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149309098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:49,102 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_1' in 994773e29f6a48e9039005d3a6da70cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:49,102 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 994773e29f6a48e9039005d3a6da70cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:49,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:49,103 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_1' in 994773e29f6a48e9039005d3a6da70cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:49,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:49,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149309108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:49,113 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 994773e29f6a48e9039005d3a6da70cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:49,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149309228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:49,497 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/7cc9b2c6ebcb4c94a48c2935a34ee258 2024-12-02T14:20:49,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/5242addf753a4f2e86ed4325f5e428be as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/5242addf753a4f2e86ed4325f5e428be 2024-12-02T14:20:49,507 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/5242addf753a4f2e86ed4325f5e428be, entries=150, sequenceid=356, filesize=30.5 K 2024-12-02T14:20:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/4ea0f2ee2be54469b906ae7e23338c8f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/4ea0f2ee2be54469b906ae7e23338c8f 2024-12-02T14:20:49,515 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/4ea0f2ee2be54469b906ae7e23338c8f, entries=150, sequenceid=356, filesize=12.0 K 2024-12-02T14:20:49,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/7cc9b2c6ebcb4c94a48c2935a34ee258 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7cc9b2c6ebcb4c94a48c2935a34ee258 2024-12-02T14:20:49,518 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7cc9b2c6ebcb4c94a48c2935a34ee258, entries=150, sequenceid=356, filesize=12.0 K 2024-12-02T14:20:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T14:20:49,519 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 994773e29f6a48e9039005d3a6da70cd in 1687ms, sequenceid=356, compaction requested=false 2024-12-02T14:20:49,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:49,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:49,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-02T14:20:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-02T14:20:49,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-02T14:20:49,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1460 sec 2024-12-02T14:20:49,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.1500 sec 2024-12-02T14:20:50,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:50,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T14:20:50,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:50,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:50,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:50,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:50,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:50,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:50,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202096494e42b9a46eab84db1f93b232537_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149250250/Put/seqid=0 2024-12-02T14:20:50,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742221_1397 (size=14994) 2024-12-02T14:20:50,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:50,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149310306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:50,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:50,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:50,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149310418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:50,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149310628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:50,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:50,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:50,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:50,662 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:50,665 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202096494e42b9a46eab84db1f93b232537_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202096494e42b9a46eab84db1f93b232537_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:50,665 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/79ba2f0c1f3c4f54924cccd526b0d4bf, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:50,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/79ba2f0c1f3c4f54924cccd526b0d4bf is 175, key is test_row_0/A:col10/1733149250250/Put/seqid=0 2024-12-02T14:20:50,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742222_1398 (size=39949) 2024-12-02T14:20:50,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149310938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:51,069 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=378, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/79ba2f0c1f3c4f54924cccd526b0d4bf 2024-12-02T14:20:51,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/a2d4667aa89948138df85fa8f3ccb7fc is 50, key is test_row_0/B:col10/1733149250250/Put/seqid=0 2024-12-02T14:20:51,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742223_1399 (size=12301) 2024-12-02T14:20:51,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37362 deadline: 1733149311109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:51,117 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_1' in 994773e29f6a48e9039005d3a6da70cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:51,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:51,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:51,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:51,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37394 deadline: 1733149311118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:51,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37426 deadline: 1733149311118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:51,121 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_1' in 994773e29f6a48e9039005d3a6da70cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:51,122 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 994773e29f6a48e9039005d3a6da70cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:51,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37358 deadline: 1733149311128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:51,134 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 994773e29f6a48e9039005d3a6da70cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:51,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:51,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:51,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:51,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149311449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:51,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:51,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:51,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/a2d4667aa89948138df85fa8f3ccb7fc 2024-12-02T14:20:51,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/128447aa59414499ae2bad86bc415a98 is 50, key is test_row_0/C:col10/1733149250250/Put/seqid=0 2024-12-02T14:20:51,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742224_1400 (size=12301) 2024-12-02T14:20:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T14:20:51,529 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:51,529 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:20:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-02T14:20:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T14:20:51,531 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:20:51,531 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:20:51,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:20:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T14:20:51,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T14:20:51,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:51,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:51,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:51,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:51,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:51,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T14:20:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:51,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:51,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T14:20:51,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/128447aa59414499ae2bad86bc415a98 2024-12-02T14:20:51,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/79ba2f0c1f3c4f54924cccd526b0d4bf as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/79ba2f0c1f3c4f54924cccd526b0d4bf 2024-12-02T14:20:51,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/79ba2f0c1f3c4f54924cccd526b0d4bf, entries=200, sequenceid=378, filesize=39.0 K 2024-12-02T14:20:51,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/a2d4667aa89948138df85fa8f3ccb7fc as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/a2d4667aa89948138df85fa8f3ccb7fc 2024-12-02T14:20:51,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/a2d4667aa89948138df85fa8f3ccb7fc, entries=150, sequenceid=378, filesize=12.0 K 2024-12-02T14:20:51,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/128447aa59414499ae2bad86bc415a98 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/128447aa59414499ae2bad86bc415a98 2024-12-02T14:20:51,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/128447aa59414499ae2bad86bc415a98, entries=150, sequenceid=378, filesize=12.0 K 2024-12-02T14:20:51,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 994773e29f6a48e9039005d3a6da70cd in 1651ms, sequenceid=378, compaction requested=true 2024-12-02T14:20:51,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:51,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:20:51,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:51,902 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:51,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:20:51,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:51,902 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:51,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:20:51,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:51,903 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:51,903 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:51,903 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/B is initiating minor compaction (all files) 2024-12-02T14:20:51,903 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/A is initiating minor compaction (all files) 2024-12-02T14:20:51,903 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/B in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:51,903 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/A in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:51,903 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/5069ae3689ef4a7ba18bda3f333d5bb1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/4ea0f2ee2be54469b906ae7e23338c8f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/a2d4667aa89948138df85fa8f3ccb7fc] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=36.5 K 2024-12-02T14:20:51,903 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/0d71dc895b3f47ed96ad92f6818efad7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/5242addf753a4f2e86ed4325f5e428be, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/79ba2f0c1f3c4f54924cccd526b0d4bf] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=100.6 K 2024-12-02T14:20:51,904 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:51,904 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/0d71dc895b3f47ed96ad92f6818efad7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/5242addf753a4f2e86ed4325f5e428be, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/79ba2f0c1f3c4f54924cccd526b0d4bf] 2024-12-02T14:20:51,904 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 5069ae3689ef4a7ba18bda3f333d5bb1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733149246889 2024-12-02T14:20:51,904 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d71dc895b3f47ed96ad92f6818efad7, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733149246889 2024-12-02T14:20:51,904 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ea0f2ee2be54469b906ae7e23338c8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733149246927 2024-12-02T14:20:51,904 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5242addf753a4f2e86ed4325f5e428be, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733149246927 2024-12-02T14:20:51,904 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting a2d4667aa89948138df85fa8f3ccb7fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733149248076 2024-12-02T14:20:51,905 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79ba2f0c1f3c4f54924cccd526b0d4bf, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733149248069 2024-12-02T14:20:51,911 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:51,912 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#B#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:51,912 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/9b9e7c9c8aa04fa2855c387c68cba277 is 50, key is test_row_0/B:col10/1733149250250/Put/seqid=0 2024-12-02T14:20:51,912 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412020e210d100f594884ade654a0f1d59e76_994773e29f6a48e9039005d3a6da70cd store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:51,914 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412020e210d100f594884ade654a0f1d59e76_994773e29f6a48e9039005d3a6da70cd, store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:51,914 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020e210d100f594884ade654a0f1d59e76_994773e29f6a48e9039005d3a6da70cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:51,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742225_1401 (size=12915) 2024-12-02T14:20:51,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742226_1402 (size=4469) 2024-12-02T14:20:51,931 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#A#compaction#334 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:51,931 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/9b9e7c9c8aa04fa2855c387c68cba277 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9b9e7c9c8aa04fa2855c387c68cba277 2024-12-02T14:20:51,931 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/8bc6918f9741499e97044c78a3b621b5 is 175, key is test_row_0/A:col10/1733149250250/Put/seqid=0 2024-12-02T14:20:51,939 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/B of 994773e29f6a48e9039005d3a6da70cd into 9b9e7c9c8aa04fa2855c387c68cba277(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:51,939 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:51,939 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/B, priority=13, startTime=1733149251902; duration=0sec 2024-12-02T14:20:51,939 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:51,940 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:B 2024-12-02T14:20:51,940 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:51,940 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:51,940 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/C is initiating minor compaction (all files) 2024-12-02T14:20:51,941 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/C in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:51,941 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/b83c57aedf2742aba90eaba3bd811e94, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7cc9b2c6ebcb4c94a48c2935a34ee258, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/128447aa59414499ae2bad86bc415a98] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=36.5 K 2024-12-02T14:20:51,941 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting b83c57aedf2742aba90eaba3bd811e94, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733149246889 2024-12-02T14:20:51,941 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cc9b2c6ebcb4c94a48c2935a34ee258, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733149246927 2024-12-02T14:20:51,943 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 128447aa59414499ae2bad86bc415a98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733149248076 2024-12-02T14:20:51,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742227_1403 (size=31869) 2024-12-02T14:20:51,950 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#C#compaction#335 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:51,950 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/7a69bc2d5241470bb52c35293ffe7e33 is 50, key is test_row_0/C:col10/1733149250250/Put/seqid=0 2024-12-02T14:20:51,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742228_1404 (size=12915) 2024-12-02T14:20:51,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T14:20:51,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:51,988 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T14:20:51,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:51,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:51,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:51,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:51,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:51,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:51,991 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/7a69bc2d5241470bb52c35293ffe7e33 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7a69bc2d5241470bb52c35293ffe7e33 2024-12-02T14:20:51,998 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/C of 994773e29f6a48e9039005d3a6da70cd into 7a69bc2d5241470bb52c35293ffe7e33(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:51,998 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:51,999 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/C, priority=13, startTime=1733149251903; duration=0sec 2024-12-02T14:20:51,999 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:51,999 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:C 2024-12-02T14:20:52,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120247fe20ac85ca4553a343c55ea21fb371_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149250274/Put/seqid=0 2024-12-02T14:20:52,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742229_1405 (size=12454) 2024-12-02T14:20:52,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:52,037 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120247fe20ac85ca4553a343c55ea21fb371_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120247fe20ac85ca4553a343c55ea21fb371_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:52,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/61f5a0720e93466f8574bfd82f256667, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:52,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/61f5a0720e93466f8574bfd82f256667 is 175, key is test_row_0/A:col10/1733149250274/Put/seqid=0 2024-12-02T14:20:52,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742230_1406 (size=31255) 2024-12-02T14:20:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T14:20:52,349 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/8bc6918f9741499e97044c78a3b621b5 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/8bc6918f9741499e97044c78a3b621b5 2024-12-02T14:20:52,352 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/A of 994773e29f6a48e9039005d3a6da70cd into 8bc6918f9741499e97044c78a3b621b5(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:52,352 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:52,352 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/A, priority=13, startTime=1733149251902; duration=0sec 2024-12-02T14:20:52,352 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:52,352 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:A 2024-12-02T14:20:52,451 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=392, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/61f5a0720e93466f8574bfd82f256667 2024-12-02T14:20:52,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/9efbba66f66c400fb57be3f569551f2f is 50, key is test_row_0/B:col10/1733149250274/Put/seqid=0 2024-12-02T14:20:52,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742231_1407 (size=12301) 2024-12-02T14:20:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:52,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. as already flushing 2024-12-02T14:20:52,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149312536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:52,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:52,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:52,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:52,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149312648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:52,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:52,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:52,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T14:20:52,859 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/9efbba66f66c400fb57be3f569551f2f 2024-12-02T14:20:52,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/ba9bb468c33a4360b92fe679d20b9fab is 50, key is test_row_0/C:col10/1733149250274/Put/seqid=0 2024-12-02T14:20:52,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742232_1408 (size=12301) 2024-12-02T14:20:52,871 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/ba9bb468c33a4360b92fe679d20b9fab 2024-12-02T14:20:52,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/61f5a0720e93466f8574bfd82f256667 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/61f5a0720e93466f8574bfd82f256667 2024-12-02T14:20:52,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149312868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:52,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:52,881 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/61f5a0720e93466f8574bfd82f256667, entries=150, sequenceid=392, filesize=30.5 K 2024-12-02T14:20:52,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/9efbba66f66c400fb57be3f569551f2f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9efbba66f66c400fb57be3f569551f2f 2024-12-02T14:20:52,887 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9efbba66f66c400fb57be3f569551f2f, entries=150, sequenceid=392, filesize=12.0 K 2024-12-02T14:20:52,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/ba9bb468c33a4360b92fe679d20b9fab as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ba9bb468c33a4360b92fe679d20b9fab 2024-12-02T14:20:52,890 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ba9bb468c33a4360b92fe679d20b9fab, entries=150, sequenceid=392, filesize=12.0 K 2024-12-02T14:20:52,891 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 994773e29f6a48e9039005d3a6da70cd in 903ms, sequenceid=392, compaction requested=false 2024-12-02T14:20:52,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:52,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:52,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-02T14:20:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-02T14:20:52,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-02T14:20:52,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3610 sec 2024-12-02T14:20:52,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.3650 sec 2024-12-02T14:20:53,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:53,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T14:20:53,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:53,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:53,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:53,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:53,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:53,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:53,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e9f983d46aec4c60a593fe46ce802c4b_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149252535/Put/seqid=0 2024-12-02T14:20:53,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742233_1409 (size=14994) 2024-12-02T14:20:53,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:53,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149313259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:53,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:53,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:53,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:53,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149313378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:53,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149313589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:53,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:53,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:53,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:53,604 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:53,607 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e9f983d46aec4c60a593fe46ce802c4b_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e9f983d46aec4c60a593fe46ce802c4b_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:53,608 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/ac37b2ec607c4a6e8bc9933d62da5cc2, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:53,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/ac37b2ec607c4a6e8bc9933d62da5cc2 is 175, key is test_row_0/A:col10/1733149252535/Put/seqid=0 2024-12-02T14:20:53,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742234_1410 (size=39949) 2024-12-02T14:20:53,634 INFO [Thread-1511 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1511. 2024-12-02T14:20:53,634 DEBUG [Thread-1511 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:53,634 DEBUG [Thread-1511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:53,634 DEBUG [Thread-1511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:53,634 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:53,637 INFO [Thread-1509 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1509. 2024-12-02T14:20:53,637 DEBUG [Thread-1509 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:53,637 DEBUG [Thread-1509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:53,637 DEBUG [Thread-1509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:53,637 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:53,639 INFO [Thread-1513 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1513. 2024-12-02T14:20:53,640 DEBUG [Thread-1513 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:53,640 DEBUG [Thread-1513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:53,640 DEBUG [Thread-1513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:53,640 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:53,642 INFO [Thread-1515 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1515. 2024-12-02T14:20:53,642 DEBUG [Thread-1515 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:53,642 DEBUG [Thread-1515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:53,642 DEBUG [Thread-1515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:53,642 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:53,644 INFO [Thread-1517 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1517. 2024-12-02T14:20:53,644 DEBUG [Thread-1517 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:53,644 DEBUG [Thread-1517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:53,644 DEBUG [Thread-1517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:53,644 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T14:20:53,679 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:53,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149313909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:54,014 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=414, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/ac37b2ec607c4a6e8bc9933d62da5cc2 2024-12-02T14:20:54,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/af7d3caccae64454a27f909b0a92b6fd is 50, key is test_row_0/B:col10/1733149252535/Put/seqid=0 2024-12-02T14:20:54,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742235_1411 (size=12301) 2024-12-02T14:20:54,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37418 deadline: 1733149314420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:54,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:54,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:54,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:20:54,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/af7d3caccae64454a27f909b0a92b6fd 2024-12-02T14:20:54,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/d680ccbccc724260a544a876d6990b9a is 50, key is test_row_0/C:col10/1733149252535/Put/seqid=0 2024-12-02T14:20:54,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742236_1412 (size=12301) 2024-12-02T14:20:54,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/d680ccbccc724260a544a876d6990b9a 2024-12-02T14:20:54,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/ac37b2ec607c4a6e8bc9933d62da5cc2 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/ac37b2ec607c4a6e8bc9933d62da5cc2 2024-12-02T14:20:54,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/ac37b2ec607c4a6e8bc9933d62da5cc2, entries=200, sequenceid=414, filesize=39.0 K 2024-12-02T14:20:54,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/af7d3caccae64454a27f909b0a92b6fd as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/af7d3caccae64454a27f909b0a92b6fd 2024-12-02T14:20:54,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/af7d3caccae64454a27f909b0a92b6fd, entries=150, sequenceid=414, filesize=12.0 K 2024-12-02T14:20:54,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/d680ccbccc724260a544a876d6990b9a as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/d680ccbccc724260a544a876d6990b9a 2024-12-02T14:20:54,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/d680ccbccc724260a544a876d6990b9a, entries=150, sequenceid=414, filesize=12.0 K 2024-12-02T14:20:54,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 994773e29f6a48e9039005d3a6da70cd in 1670ms, sequenceid=414, compaction requested=true 2024-12-02T14:20:54,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:54,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:20:54,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:54,865 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:54,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:20:54,866 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:54,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:54,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 994773e29f6a48e9039005d3a6da70cd:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:20:54,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:54,866 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:54,866 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:54,867 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/A is initiating minor compaction (all files) 2024-12-02T14:20:54,867 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/B is initiating minor compaction (all files) 2024-12-02T14:20:54,867 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/B in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:54,867 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/A in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:54,867 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9b9e7c9c8aa04fa2855c387c68cba277, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9efbba66f66c400fb57be3f569551f2f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/af7d3caccae64454a27f909b0a92b6fd] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=36.6 K 2024-12-02T14:20:54,867 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/8bc6918f9741499e97044c78a3b621b5, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/61f5a0720e93466f8574bfd82f256667, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/ac37b2ec607c4a6e8bc9933d62da5cc2] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=100.7 K 2024-12-02T14:20:54,867 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:54,867 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/8bc6918f9741499e97044c78a3b621b5, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/61f5a0720e93466f8574bfd82f256667, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/ac37b2ec607c4a6e8bc9933d62da5cc2] 2024-12-02T14:20:54,867 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b9e7c9c8aa04fa2855c387c68cba277, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733149248076 2024-12-02T14:20:54,867 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bc6918f9741499e97044c78a3b621b5, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733149248076 2024-12-02T14:20:54,867 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 9efbba66f66c400fb57be3f569551f2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733149250262 2024-12-02T14:20:54,867 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61f5a0720e93466f8574bfd82f256667, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733149250262 2024-12-02T14:20:54,868 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting af7d3caccae64454a27f909b0a92b6fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733149252511 2024-12-02T14:20:54,868 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac37b2ec607c4a6e8bc9933d62da5cc2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733149252501 2024-12-02T14:20:54,874 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:54,874 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:54,874 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/848311f59677400b94ce4cb00ed895f5 is 50, key is test_row_0/B:col10/1733149252535/Put/seqid=0 2024-12-02T14:20:54,876 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202a8c2907feedb4893b871a3626988e170_994773e29f6a48e9039005d3a6da70cd store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:54,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742237_1413 (size=13017) 2024-12-02T14:20:54,884 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202a8c2907feedb4893b871a3626988e170_994773e29f6a48e9039005d3a6da70cd, store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:54,884 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a8c2907feedb4893b871a3626988e170_994773e29f6a48e9039005d3a6da70cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:54,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742238_1414 (size=4469) 2024-12-02T14:20:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:55,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T14:20:55,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:55,131 INFO [Thread-1502 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1502. 2024-12-02T14:20:55,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:55,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:55,131 DEBUG [Thread-1502 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:55,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:55,131 DEBUG [Thread-1502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:55,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:55,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:55,132 DEBUG [Thread-1502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:55,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:55,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027eccb31d57db4d7781f5d6c27b5a5d02_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149253252/Put/seqid=0 2024-12-02T14:20:55,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742239_1415 (size=12454) 2024-12-02T14:20:55,149 INFO [Thread-1500 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1500. 2024-12-02T14:20:55,149 INFO [Thread-1504 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1504. 2024-12-02T14:20:55,150 DEBUG [Thread-1504 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:55,150 DEBUG [Thread-1500 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:55,150 DEBUG [Thread-1504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:55,150 DEBUG [Thread-1500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:55,150 DEBUG [Thread-1500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:55,150 DEBUG [Thread-1504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:55,150 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:55,150 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:55,160 INFO [Thread-1506 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1506. 2024-12-02T14:20:55,160 DEBUG [Thread-1506 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:55,160 DEBUG [Thread-1506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:55,160 DEBUG [Thread-1506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:55,160 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:55,287 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/848311f59677400b94ce4cb00ed895f5 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/848311f59677400b94ce4cb00ed895f5 2024-12-02T14:20:55,289 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#A#compaction#343 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:55,289 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/8af1bf9c434d49feaaae96ed8085b8dd is 175, key is test_row_0/A:col10/1733149252535/Put/seqid=0 2024-12-02T14:20:55,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742240_1416 (size=31971) 2024-12-02T14:20:55,292 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/B of 994773e29f6a48e9039005d3a6da70cd into 848311f59677400b94ce4cb00ed895f5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:55,292 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:55,292 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/B, priority=13, startTime=1733149254865; duration=0sec 2024-12-02T14:20:55,292 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:20:55,292 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:B 2024-12-02T14:20:55,292 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:20:55,293 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:20:55,293 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 994773e29f6a48e9039005d3a6da70cd/C is initiating minor compaction (all files) 2024-12-02T14:20:55,293 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 994773e29f6a48e9039005d3a6da70cd/C in TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:55,293 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7a69bc2d5241470bb52c35293ffe7e33, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ba9bb468c33a4360b92fe679d20b9fab, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/d680ccbccc724260a544a876d6990b9a] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp, totalSize=36.6 K 2024-12-02T14:20:55,294 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a69bc2d5241470bb52c35293ffe7e33, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1733149248076 2024-12-02T14:20:55,294 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting ba9bb468c33a4360b92fe679d20b9fab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733149250262 2024-12-02T14:20:55,294 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting d680ccbccc724260a544a876d6990b9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1733149252511 2024-12-02T14:20:55,300 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 994773e29f6a48e9039005d3a6da70cd#C#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:20:55,300 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/9a5575d5b7f2436ea6af6cbbe6201923 is 50, key is test_row_0/C:col10/1733149252535/Put/seqid=0 2024-12-02T14:20:55,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742241_1417 (size=13017) 2024-12-02T14:20:55,430 INFO [Thread-1498 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1498. 2024-12-02T14:20:55,430 DEBUG [Thread-1498 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:20:55,430 DEBUG [Thread-1498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:55,430 DEBUG [Thread-1498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:55,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:55,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T14:20:55,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-12-02T14:20:55,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-12-02T14:20:55,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-12-02T14:20:55,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-12-02T14:20:55,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-12-02T14:20:55,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T14:20:55,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T14:20:55,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1455 2024-12-02T14:20:55,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4365 rows 2024-12-02T14:20:55,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1440 2024-12-02T14:20:55,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4314 rows 2024-12-02T14:20:55,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1430 2024-12-02T14:20:55,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4290 rows 2024-12-02T14:20:55,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1445 2024-12-02T14:20:55,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4334 rows 2024-12-02T14:20:55,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1452 2024-12-02T14:20:55,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4352 rows 2024-12-02T14:20:55,431 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T14:20:55,431 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T14:20:55,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:55,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:55,431 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:20:55,432 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T14:20:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T14:20:55,434 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149255434"}]},"ts":"1733149255434"} 2024-12-02T14:20:55,435 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T14:20:55,436 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T14:20:55,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T14:20:55,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=994773e29f6a48e9039005d3a6da70cd, UNASSIGN}] 2024-12-02T14:20:55,438 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=994773e29f6a48e9039005d3a6da70cd, UNASSIGN 2024-12-02T14:20:55,439 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=994773e29f6a48e9039005d3a6da70cd, regionState=CLOSING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:20:55,440 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=994773e29f6a48e9039005d3a6da70cd, UNASSIGN because future has completed 2024-12-02T14:20:55,440 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T14:20:55,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:20:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T14:20:55,547 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:55,553 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027eccb31d57db4d7781f5d6c27b5a5d02_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027eccb31d57db4d7781f5d6c27b5a5d02_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:55,555 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/700ed0c8f6104941861ae3088ba0daf1, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:55,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/700ed0c8f6104941861ae3088ba0daf1 is 175, key is test_row_0/A:col10/1733149253252/Put/seqid=0 2024-12-02T14:20:55,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742242_1418 (size=31255) 2024-12-02T14:20:55,593 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:55,593 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T14:20:55,593 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 994773e29f6a48e9039005d3a6da70cd, disabling compactions & flushes 2024-12-02T14:20:55,593 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1993): waiting for 2 compactions & cache flush to complete for region TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:55,703 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/8af1bf9c434d49feaaae96ed8085b8dd as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/8af1bf9c434d49feaaae96ed8085b8dd 2024-12-02T14:20:55,709 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/A of 994773e29f6a48e9039005d3a6da70cd into 8af1bf9c434d49feaaae96ed8085b8dd(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:55,709 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:55,709 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/A, priority=13, startTime=1733149254865; duration=0sec 2024-12-02T14:20:55,709 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:55,709 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:A 2024-12-02T14:20:55,710 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/9a5575d5b7f2436ea6af6cbbe6201923 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/9a5575d5b7f2436ea6af6cbbe6201923 2024-12-02T14:20:55,713 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 994773e29f6a48e9039005d3a6da70cd/C of 994773e29f6a48e9039005d3a6da70cd into 9a5575d5b7f2436ea6af6cbbe6201923(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:20:55,713 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:55,713 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., storeName=994773e29f6a48e9039005d3a6da70cd/C, priority=13, startTime=1733149254866; duration=0sec 2024-12-02T14:20:55,713 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:55,713 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:20:55,713 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 994773e29f6a48e9039005d3a6da70cd:C 2024-12-02T14:20:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T14:20:55,962 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=427, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/700ed0c8f6104941861ae3088ba0daf1 2024-12-02T14:20:55,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/2a9bd80fabf440a49ae7e9779ba2e831 is 50, key is test_row_0/B:col10/1733149253252/Put/seqid=0 2024-12-02T14:20:55,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742243_1419 (size=12301) 2024-12-02T14:20:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T14:20:56,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/2a9bd80fabf440a49ae7e9779ba2e831 2024-12-02T14:20:56,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/7115eb0da05b4a30b5c65cd38294a78e is 50, key is test_row_0/C:col10/1733149253252/Put/seqid=0 2024-12-02T14:20:56,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742244_1420 (size=12301) 2024-12-02T14:20:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T14:20:56,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/7115eb0da05b4a30b5c65cd38294a78e 2024-12-02T14:20:56,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/700ed0c8f6104941861ae3088ba0daf1 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/700ed0c8f6104941861ae3088ba0daf1 2024-12-02T14:20:56,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/700ed0c8f6104941861ae3088ba0daf1, entries=150, sequenceid=427, filesize=30.5 K 2024-12-02T14:20:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/2a9bd80fabf440a49ae7e9779ba2e831 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/2a9bd80fabf440a49ae7e9779ba2e831 2024-12-02T14:20:56,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/2a9bd80fabf440a49ae7e9779ba2e831, entries=150, sequenceid=427, filesize=12.0 K 2024-12-02T14:20:56,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/7115eb0da05b4a30b5c65cd38294a78e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7115eb0da05b4a30b5c65cd38294a78e 2024-12-02T14:20:56,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7115eb0da05b4a30b5c65cd38294a78e, entries=150, sequenceid=427, filesize=12.0 K 2024-12-02T14:20:56,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for 994773e29f6a48e9039005d3a6da70cd in 1686ms, sequenceid=427, compaction requested=false 2024-12-02T14:20:56,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 994773e29f6a48e9039005d3a6da70cd: 2024-12-02T14:20:56,817 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:56,817 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:56,817 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. after waiting 0 ms 2024-12-02T14:20:56,817 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:56,817 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 994773e29f6a48e9039005d3a6da70cd 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-02T14:20:56,818 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=A 2024-12-02T14:20:56,818 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:56,818 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=B 2024-12-02T14:20:56,818 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:56,818 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 994773e29f6a48e9039005d3a6da70cd, store=C 2024-12-02T14:20:56,818 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:20:56,823 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202fb4dc2019ac74b3d9c00f73fe812371a_994773e29f6a48e9039005d3a6da70cd is 50, key is test_row_0/A:col10/1733149255429/Put/seqid=0 2024-12-02T14:20:56,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742245_1421 (size=12454) 2024-12-02T14:20:57,048 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T14:20:57,227 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:57,234 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202fb4dc2019ac74b3d9c00f73fe812371a_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fb4dc2019ac74b3d9c00f73fe812371a_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:57,236 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/814f089191ff466a905d7ad24dd23dd5, store: [table=TestAcidGuarantees family=A region=994773e29f6a48e9039005d3a6da70cd] 2024-12-02T14:20:57,237 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/814f089191ff466a905d7ad24dd23dd5 is 175, key is test_row_0/A:col10/1733149255429/Put/seqid=0 2024-12-02T14:20:57,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742246_1422 (size=31255) 2024-12-02T14:20:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T14:20:57,643 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=437, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/814f089191ff466a905d7ad24dd23dd5 2024-12-02T14:20:57,654 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/2ab0852bda564d4988e18652cecee565 is 50, key is test_row_0/B:col10/1733149255429/Put/seqid=0 2024-12-02T14:20:57,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742247_1423 (size=12301) 2024-12-02T14:20:58,058 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/2ab0852bda564d4988e18652cecee565 2024-12-02T14:20:58,065 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/b60f1cb89b9f459b8a378a80e5539627 is 50, key is test_row_0/C:col10/1733149255429/Put/seqid=0 2024-12-02T14:20:58,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742248_1424 (size=12301) 2024-12-02T14:20:58,469 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/b60f1cb89b9f459b8a378a80e5539627 2024-12-02T14:20:58,474 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/A/814f089191ff466a905d7ad24dd23dd5 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/814f089191ff466a905d7ad24dd23dd5 2024-12-02T14:20:58,479 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/814f089191ff466a905d7ad24dd23dd5, entries=150, sequenceid=437, filesize=30.5 K 2024-12-02T14:20:58,480 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/B/2ab0852bda564d4988e18652cecee565 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/2ab0852bda564d4988e18652cecee565 2024-12-02T14:20:58,485 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/2ab0852bda564d4988e18652cecee565, entries=150, sequenceid=437, filesize=12.0 K 2024-12-02T14:20:58,486 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/.tmp/C/b60f1cb89b9f459b8a378a80e5539627 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/b60f1cb89b9f459b8a378a80e5539627 2024-12-02T14:20:58,490 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/b60f1cb89b9f459b8a378a80e5539627, entries=150, sequenceid=437, filesize=12.0 K 2024-12-02T14:20:58,491 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 994773e29f6a48e9039005d3a6da70cd in 1674ms, sequenceid=437, compaction requested=true 2024-12-02T14:20:58,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/eb08a389e003443daacc17f4d496c293, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/ba89e28286504939b5d9ffd1911173ef, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/5fc3d4d169a047e8a81cb376139961f3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/70349863ec654619900bf5d86bb22608, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/660db7534288460795658cc53b56f49a, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/f9281e00e19349b9bbcf8e6ad4f4733f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/d771aa0294724b07a899d020d0174a2e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/563c1db2a09a42ff8833e0bfc7d459d1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/b870db388c4e48b7b44f26e15f6bbfe5, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/b866d2e02f0f45ada7cb6e8bf79640ac, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/e4ca5bc3155347d58b13e850025d8990, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/a78912245cfb4a348b7805dea1548010, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/aeacf69e605147a3a16a9ec86dc39f52, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/67679d1f67024e909deead690e1492d9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/38472b30296a49f58f67da16ed623f4f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/0d71dc895b3f47ed96ad92f6818efad7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/5242addf753a4f2e86ed4325f5e428be, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/79ba2f0c1f3c4f54924cccd526b0d4bf, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/8bc6918f9741499e97044c78a3b621b5, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/61f5a0720e93466f8574bfd82f256667, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/ac37b2ec607c4a6e8bc9933d62da5cc2] to archive 2024-12-02T14:20:58,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:20:58,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/eb08a389e003443daacc17f4d496c293 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/eb08a389e003443daacc17f4d496c293 2024-12-02T14:20:58,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/ba89e28286504939b5d9ffd1911173ef to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/ba89e28286504939b5d9ffd1911173ef 2024-12-02T14:20:58,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/5fc3d4d169a047e8a81cb376139961f3 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/5fc3d4d169a047e8a81cb376139961f3 2024-12-02T14:20:58,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/70349863ec654619900bf5d86bb22608 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/70349863ec654619900bf5d86bb22608 2024-12-02T14:20:58,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/660db7534288460795658cc53b56f49a to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/660db7534288460795658cc53b56f49a 2024-12-02T14:20:58,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/f9281e00e19349b9bbcf8e6ad4f4733f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/f9281e00e19349b9bbcf8e6ad4f4733f 2024-12-02T14:20:58,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/d771aa0294724b07a899d020d0174a2e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/d771aa0294724b07a899d020d0174a2e 2024-12-02T14:20:58,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/563c1db2a09a42ff8833e0bfc7d459d1 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/563c1db2a09a42ff8833e0bfc7d459d1 2024-12-02T14:20:58,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/b870db388c4e48b7b44f26e15f6bbfe5 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/b870db388c4e48b7b44f26e15f6bbfe5 2024-12-02T14:20:58,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/b866d2e02f0f45ada7cb6e8bf79640ac to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/b866d2e02f0f45ada7cb6e8bf79640ac 2024-12-02T14:20:58,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/e4ca5bc3155347d58b13e850025d8990 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/e4ca5bc3155347d58b13e850025d8990 2024-12-02T14:20:58,502 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/a78912245cfb4a348b7805dea1548010 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/a78912245cfb4a348b7805dea1548010 2024-12-02T14:20:58,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/aeacf69e605147a3a16a9ec86dc39f52 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/aeacf69e605147a3a16a9ec86dc39f52 2024-12-02T14:20:58,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/67679d1f67024e909deead690e1492d9 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/67679d1f67024e909deead690e1492d9 2024-12-02T14:20:58,505 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/38472b30296a49f58f67da16ed623f4f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/38472b30296a49f58f67da16ed623f4f 2024-12-02T14:20:58,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/0d71dc895b3f47ed96ad92f6818efad7 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/0d71dc895b3f47ed96ad92f6818efad7 2024-12-02T14:20:58,507 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/5242addf753a4f2e86ed4325f5e428be to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/5242addf753a4f2e86ed4325f5e428be 2024-12-02T14:20:58,508 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/79ba2f0c1f3c4f54924cccd526b0d4bf to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/79ba2f0c1f3c4f54924cccd526b0d4bf 2024-12-02T14:20:58,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/8bc6918f9741499e97044c78a3b621b5 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/8bc6918f9741499e97044c78a3b621b5 2024-12-02T14:20:58,511 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/61f5a0720e93466f8574bfd82f256667 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/61f5a0720e93466f8574bfd82f256667 2024-12-02T14:20:58,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/ac37b2ec607c4a6e8bc9933d62da5cc2 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/ac37b2ec607c4a6e8bc9933d62da5cc2 2024-12-02T14:20:58,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/991ad938bff142e6b995e950909f92d1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/b064309d99074f529796b86c7fd88f12, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/76e6cce1f598427888b876625bdb83c9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/aaefbe4272c94c15a90fa613c429cd31, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/8e2bce1d11d8418ebefb8b14caf30398, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/b5b2476116c04b6fa8c52a9ad6880683, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9fc34163a4d64715b213213c3602340e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/4f0033ab7ee94526b060991322eb475f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/f8b4888a3d124514a68dfbaca1ebb490, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/6ac502bd478544adb70e21a319795516, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/d9f9210c1dfb4c039bec1a01733542ba, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/6deafd897fd74839a20fe0cb3b535e95, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9646e64c938e44c39f8ac78fada582e9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/a34b5372acae4be28f94b62eaf440029, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/5069ae3689ef4a7ba18bda3f333d5bb1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/395c627f0825436780d989098a655459, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/4ea0f2ee2be54469b906ae7e23338c8f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9b9e7c9c8aa04fa2855c387c68cba277, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/a2d4667aa89948138df85fa8f3ccb7fc, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9efbba66f66c400fb57be3f569551f2f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/af7d3caccae64454a27f909b0a92b6fd] to archive 2024-12-02T14:20:58,514 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:20:58,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/991ad938bff142e6b995e950909f92d1 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/991ad938bff142e6b995e950909f92d1 2024-12-02T14:20:58,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/b064309d99074f529796b86c7fd88f12 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/b064309d99074f529796b86c7fd88f12 2024-12-02T14:20:58,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/76e6cce1f598427888b876625bdb83c9 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/76e6cce1f598427888b876625bdb83c9 2024-12-02T14:20:58,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/aaefbe4272c94c15a90fa613c429cd31 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/aaefbe4272c94c15a90fa613c429cd31 2024-12-02T14:20:58,520 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/8e2bce1d11d8418ebefb8b14caf30398 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/8e2bce1d11d8418ebefb8b14caf30398 2024-12-02T14:20:58,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/b5b2476116c04b6fa8c52a9ad6880683 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/b5b2476116c04b6fa8c52a9ad6880683 2024-12-02T14:20:58,522 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9fc34163a4d64715b213213c3602340e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9fc34163a4d64715b213213c3602340e 2024-12-02T14:20:58,523 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/4f0033ab7ee94526b060991322eb475f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/4f0033ab7ee94526b060991322eb475f 2024-12-02T14:20:58,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/f8b4888a3d124514a68dfbaca1ebb490 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/f8b4888a3d124514a68dfbaca1ebb490 2024-12-02T14:20:58,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/6ac502bd478544adb70e21a319795516 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/6ac502bd478544adb70e21a319795516 2024-12-02T14:20:58,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/d9f9210c1dfb4c039bec1a01733542ba to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/d9f9210c1dfb4c039bec1a01733542ba 2024-12-02T14:20:58,526 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/6deafd897fd74839a20fe0cb3b535e95 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/6deafd897fd74839a20fe0cb3b535e95 2024-12-02T14:20:58,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9646e64c938e44c39f8ac78fada582e9 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9646e64c938e44c39f8ac78fada582e9 2024-12-02T14:20:58,528 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/a34b5372acae4be28f94b62eaf440029 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/a34b5372acae4be28f94b62eaf440029 2024-12-02T14:20:58,529 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/5069ae3689ef4a7ba18bda3f333d5bb1 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/5069ae3689ef4a7ba18bda3f333d5bb1 2024-12-02T14:20:58,530 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/395c627f0825436780d989098a655459 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/395c627f0825436780d989098a655459 2024-12-02T14:20:58,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/4ea0f2ee2be54469b906ae7e23338c8f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/4ea0f2ee2be54469b906ae7e23338c8f 2024-12-02T14:20:58,532 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9b9e7c9c8aa04fa2855c387c68cba277 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9b9e7c9c8aa04fa2855c387c68cba277 2024-12-02T14:20:58,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/a2d4667aa89948138df85fa8f3ccb7fc to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/a2d4667aa89948138df85fa8f3ccb7fc 2024-12-02T14:20:58,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9efbba66f66c400fb57be3f569551f2f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/9efbba66f66c400fb57be3f569551f2f 2024-12-02T14:20:58,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/af7d3caccae64454a27f909b0a92b6fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/af7d3caccae64454a27f909b0a92b6fd 2024-12-02T14:20:58,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/f7b0daa9c0894e89b43f7fbed9f570da, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/4f79890517204c97844422134014a6c0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ec5deed2171d47a8b28b906bfbe14dbe, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/45231d57e47f40f2936c6a71265ed9b4, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/c93a7ecfcfd142a4b5648bb08668059e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7f293374c4b34d068ac1a08a5e217298, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/099ae44d52f2454f9990c3b2f4f002b8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ce71ccd687d84228ab0050771f0c16e9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/a13cc4ab99e9405097326ac563ef7d0b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/c0d9558db7dc47618d7bbba201997103, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/147536d3570d4796beee2fcedc33e1a8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/332f6dc209d64d7997f656d29dba78fa, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7e6d9fe9d5c046c3a1feafacc183b164, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/463c78ddcd7d45b0a1cb4db1c896faca, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/b83c57aedf2742aba90eaba3bd811e94, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/b461e22086ee4342ba30cafeee2c3ccd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7cc9b2c6ebcb4c94a48c2935a34ee258, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7a69bc2d5241470bb52c35293ffe7e33, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/128447aa59414499ae2bad86bc415a98, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ba9bb468c33a4360b92fe679d20b9fab, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/d680ccbccc724260a544a876d6990b9a] to archive 2024-12-02T14:20:58,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:20:58,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/f7b0daa9c0894e89b43f7fbed9f570da to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/f7b0daa9c0894e89b43f7fbed9f570da 2024-12-02T14:20:58,539 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/4f79890517204c97844422134014a6c0 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/4f79890517204c97844422134014a6c0 2024-12-02T14:20:58,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ec5deed2171d47a8b28b906bfbe14dbe to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ec5deed2171d47a8b28b906bfbe14dbe 2024-12-02T14:20:58,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/45231d57e47f40f2936c6a71265ed9b4 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/45231d57e47f40f2936c6a71265ed9b4 2024-12-02T14:20:58,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/c93a7ecfcfd142a4b5648bb08668059e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/c93a7ecfcfd142a4b5648bb08668059e 2024-12-02T14:20:58,542 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7f293374c4b34d068ac1a08a5e217298 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7f293374c4b34d068ac1a08a5e217298 2024-12-02T14:20:58,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/099ae44d52f2454f9990c3b2f4f002b8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/099ae44d52f2454f9990c3b2f4f002b8 2024-12-02T14:20:58,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ce71ccd687d84228ab0050771f0c16e9 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ce71ccd687d84228ab0050771f0c16e9 2024-12-02T14:20:58,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/a13cc4ab99e9405097326ac563ef7d0b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/a13cc4ab99e9405097326ac563ef7d0b 2024-12-02T14:20:58,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/c0d9558db7dc47618d7bbba201997103 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/c0d9558db7dc47618d7bbba201997103 2024-12-02T14:20:58,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/147536d3570d4796beee2fcedc33e1a8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/147536d3570d4796beee2fcedc33e1a8 2024-12-02T14:20:58,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/332f6dc209d64d7997f656d29dba78fa to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/332f6dc209d64d7997f656d29dba78fa 2024-12-02T14:20:58,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7e6d9fe9d5c046c3a1feafacc183b164 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7e6d9fe9d5c046c3a1feafacc183b164 2024-12-02T14:20:58,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/463c78ddcd7d45b0a1cb4db1c896faca to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/463c78ddcd7d45b0a1cb4db1c896faca 2024-12-02T14:20:58,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/b83c57aedf2742aba90eaba3bd811e94 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/b83c57aedf2742aba90eaba3bd811e94 2024-12-02T14:20:58,549 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/b461e22086ee4342ba30cafeee2c3ccd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/b461e22086ee4342ba30cafeee2c3ccd 2024-12-02T14:20:58,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7cc9b2c6ebcb4c94a48c2935a34ee258 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7cc9b2c6ebcb4c94a48c2935a34ee258 2024-12-02T14:20:58,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7a69bc2d5241470bb52c35293ffe7e33 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7a69bc2d5241470bb52c35293ffe7e33 2024-12-02T14:20:58,551 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/128447aa59414499ae2bad86bc415a98 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/128447aa59414499ae2bad86bc415a98 2024-12-02T14:20:58,551 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ba9bb468c33a4360b92fe679d20b9fab to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/ba9bb468c33a4360b92fe679d20b9fab 2024-12-02T14:20:58,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/d680ccbccc724260a544a876d6990b9a to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/d680ccbccc724260a544a876d6990b9a 2024-12-02T14:20:58,555 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/recovered.edits/440.seqid, newMaxSeqId=440, maxSeqId=4 2024-12-02T14:20:58,555 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. 2024-12-02T14:20:58,555 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 994773e29f6a48e9039005d3a6da70cd: Waiting for close lock at 1733149255593Running coprocessor pre-close hooks at 1733149255593Disabling compacts and flushes for region at 1733149255593Disabling writes for close at 1733149256817 (+1224 ms)Obtaining lock to block concurrent updates at 1733149256817Preparing flush snapshotting stores in 994773e29f6a48e9039005d3a6da70cd at 1733149256817Finished memstore snapshotting TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733149256818 (+1 ms)Flushing stores of TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd. at 1733149256818Flushing 994773e29f6a48e9039005d3a6da70cd/A: creating writer at 1733149256818Flushing mob file 994773e29f6a48e9039005d3a6da70cd/A: appending metadata at 1733149256823 (+5 ms)Flushing mob file 994773e29f6a48e9039005d3a6da70cd/A: closing flushed file at 1733149256823Flushing 994773e29f6a48e9039005d3a6da70cd/A: appending metadata at 1733149257236 (+413 ms)Flushing 994773e29f6a48e9039005d3a6da70cd/A: closing flushed file at 1733149257237 (+1 ms)Flushing 994773e29f6a48e9039005d3a6da70cd/B: creating writer at 1733149257648 (+411 ms)Flushing 994773e29f6a48e9039005d3a6da70cd/B: appending metadata at 1733149257653 (+5 ms)Flushing 994773e29f6a48e9039005d3a6da70cd/B: closing flushed file at 1733149257653Flushing 994773e29f6a48e9039005d3a6da70cd/C: creating writer at 1733149258062 (+409 ms)Flushing 994773e29f6a48e9039005d3a6da70cd/C: appending metadata at 1733149258064 (+2 ms)Flushing 994773e29f6a48e9039005d3a6da70cd/C: closing flushed file at 1733149258064Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ffb0f35: reopening flushed file at 1733149258473 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f99e0a6: reopening flushed file at 1733149258479 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b705e65: reopening flushed file at 1733149258485 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 994773e29f6a48e9039005d3a6da70cd in 1674ms, sequenceid=437, compaction requested=true at 1733149258491 (+6 ms)Writing region close event to WAL at 1733149258553 (+62 ms)Running coprocessor post-close hooks at 1733149258555 (+2 ms)Closed at 1733149258555 2024-12-02T14:20:58,557 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:58,557 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=994773e29f6a48e9039005d3a6da70cd, regionState=CLOSED 2024-12-02T14:20:58,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:20:58,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-02T14:20:58,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 994773e29f6a48e9039005d3a6da70cd, server=a3a61c9ba14f,46793,1733149138543 in 3.1190 sec 2024-12-02T14:20:58,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-12-02T14:20:58,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=994773e29f6a48e9039005d3a6da70cd, UNASSIGN in 3.1240 sec 2024-12-02T14:20:58,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-02T14:20:58,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.1270 sec 2024-12-02T14:20:58,567 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149258567"}]},"ts":"1733149258567"} 2024-12-02T14:20:58,569 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T14:20:58,569 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T14:20:58,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.1370 sec 2024-12-02T14:20:59,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T14:20:59,608 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:59,609 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T14:20:59,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:59,611 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T14:20:59,612 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:59,614 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,615 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/recovered.edits] 2024-12-02T14:20:59,617 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/700ed0c8f6104941861ae3088ba0daf1 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/700ed0c8f6104941861ae3088ba0daf1 2024-12-02T14:20:59,618 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/814f089191ff466a905d7ad24dd23dd5 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/814f089191ff466a905d7ad24dd23dd5 2024-12-02T14:20:59,619 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/8af1bf9c434d49feaaae96ed8085b8dd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/A/8af1bf9c434d49feaaae96ed8085b8dd 2024-12-02T14:20:59,621 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/2a9bd80fabf440a49ae7e9779ba2e831 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/2a9bd80fabf440a49ae7e9779ba2e831 2024-12-02T14:20:59,621 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/2ab0852bda564d4988e18652cecee565 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/2ab0852bda564d4988e18652cecee565 2024-12-02T14:20:59,622 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/848311f59677400b94ce4cb00ed895f5 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/B/848311f59677400b94ce4cb00ed895f5 2024-12-02T14:20:59,624 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7115eb0da05b4a30b5c65cd38294a78e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/7115eb0da05b4a30b5c65cd38294a78e 2024-12-02T14:20:59,625 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/9a5575d5b7f2436ea6af6cbbe6201923 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/9a5575d5b7f2436ea6af6cbbe6201923 2024-12-02T14:20:59,626 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/b60f1cb89b9f459b8a378a80e5539627 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/C/b60f1cb89b9f459b8a378a80e5539627 2024-12-02T14:20:59,628 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/recovered.edits/440.seqid to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd/recovered.edits/440.seqid 2024-12-02T14:20:59,628 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,628 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T14:20:59,629 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T14:20:59,629 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T14:20:59,632 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202057eccfec9464507bb07ddbeb3826ef1_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202057eccfec9464507bb07ddbeb3826ef1_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,633 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202096494e42b9a46eab84db1f93b232537_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202096494e42b9a46eab84db1f93b232537_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,634 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202201eddd16df54c218175f0d78b2ecbd2_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202201eddd16df54c218175f0d78b2ecbd2_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,634 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120223490b5c097640aa9773f7b73a7d72eb_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120223490b5c097640aa9773f7b73a7d72eb_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,635 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202313ecb2e29634188bc7e7fbe1986a189_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202313ecb2e29634188bc7e7fbe1986a189_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,636 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412023f43476f0d7845ca9c6cb8c62c5e3b43_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412023f43476f0d7845ca9c6cb8c62c5e3b43_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,637 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024076a8b4f4544be5bf2ef45f09d729d7_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024076a8b4f4544be5bf2ef45f09d729d7_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,638 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120247fe20ac85ca4553a343c55ea21fb371_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120247fe20ac85ca4553a343c55ea21fb371_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,639 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027eccb31d57db4d7781f5d6c27b5a5d02_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027eccb31d57db4d7781f5d6c27b5a5d02_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,640 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027fd6f1481fcf446ea41756740ca80e7e_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027fd6f1481fcf446ea41756740ca80e7e_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,641 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120288678dd1c0fc491887d5f6a162d60c1e_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120288678dd1c0fc491887d5f6a162d60c1e_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,642 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029389ba69621844e384cb01a7349581e4_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029389ba69621844e384cb01a7349581e4_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,643 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bb46a6f5d3a54be5a36f5d8bf0af1b6f_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bb46a6f5d3a54be5a36f5d8bf0af1b6f_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,644 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c95554e500aa45f28ce65af8c0c9de06_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c95554e500aa45f28ce65af8c0c9de06_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,645 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d0601d7041864ebe8a9137a4266602c5_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d0601d7041864ebe8a9137a4266602c5_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,646 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e9f983d46aec4c60a593fe46ce802c4b_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e9f983d46aec4c60a593fe46ce802c4b_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,646 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f50d8a0ea5fa4b62940368bcb8744e76_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f50d8a0ea5fa4b62940368bcb8744e76_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,647 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fb4dc2019ac74b3d9c00f73fe812371a_994773e29f6a48e9039005d3a6da70cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fb4dc2019ac74b3d9c00f73fe812371a_994773e29f6a48e9039005d3a6da70cd 2024-12-02T14:20:59,648 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T14:20:59,649 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:59,651 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T14:20:59,653 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T14:20:59,654 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:59,654 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T14:20:59,654 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733149259654"}]},"ts":"9223372036854775807"} 2024-12-02T14:20:59,656 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T14:20:59,656 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 994773e29f6a48e9039005d3a6da70cd, NAME => 'TestAcidGuarantees,,1733149230446.994773e29f6a48e9039005d3a6da70cd.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T14:20:59,656 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T14:20:59,657 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733149259656"}]},"ts":"9223372036854775807"} 2024-12-02T14:20:59,658 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T14:20:59,659 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:59,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-12-02T14:20:59,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T14:20:59,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T14:20:59,728 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:20:59,736 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 249) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_180533634_22 at /127.0.0.1:35834 [Waiting for operation #1239] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-938368545_22 at /127.0.0.1:35828 [Waiting for operation #1298] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_180533634_22 at /127.0.0.1:52886 [Waiting for operation #1537] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-938368545_22 at /127.0.0.1:35820 [Waiting for operation #1288] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=488 (was 452) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5900 (was 5841) - AvailableMemoryMB LEAK? - 2024-12-02T14:20:59,744 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=488, ProcessCount=11, AvailableMemoryMB=5899 2024-12-02T14:20:59,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T14:20:59,745 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T14:20:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T14:20:59,747 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T14:20:59,747 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:59,747 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-12-02T14:20:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-02T14:20:59,748 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T14:20:59,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742249_1425 (size=1061) 2024-12-02T14:20:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-02T14:21:00,015 WARN [regionserver/a3a61c9ba14f:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-12-02T14:21:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-02T14:21:00,154 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919 2024-12-02T14:21:00,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742250_1426 (size=53) 2024-12-02T14:21:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-02T14:21:00,560 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:21:00,560 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9c93976ab5d5804c690c7e75c9981484, disabling compactions & flushes 2024-12-02T14:21:00,560 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:00,560 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:00,560 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. after waiting 0 ms 2024-12-02T14:21:00,560 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:00,561 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:00,561 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9c93976ab5d5804c690c7e75c9981484: Waiting for close lock at 1733149260560Disabling compacts and flushes for region at 1733149260560Disabling writes for close at 1733149260560Writing region close event to WAL at 1733149260560Closed at 1733149260560 2024-12-02T14:21:00,563 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T14:21:00,563 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733149260563"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733149260563"}]},"ts":"1733149260563"} 2024-12-02T14:21:00,566 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T14:21:00,567 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T14:21:00,567 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149260567"}]},"ts":"1733149260567"} 2024-12-02T14:21:00,569 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T14:21:00,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c93976ab5d5804c690c7e75c9981484, ASSIGN}] 2024-12-02T14:21:00,572 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c93976ab5d5804c690c7e75c9981484, ASSIGN 2024-12-02T14:21:00,573 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c93976ab5d5804c690c7e75c9981484, ASSIGN; state=OFFLINE, location=a3a61c9ba14f,46793,1733149138543; forceNewPlan=false, retain=false 2024-12-02T14:21:00,724 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=9c93976ab5d5804c690c7e75c9981484, regionState=OPENING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:00,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c93976ab5d5804c690c7e75c9981484, ASSIGN because future has completed 2024-12-02T14:21:00,727 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:21:00,885 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:00,885 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} 2024-12-02T14:21:00,886 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:00,887 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:21:00,887 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:00,887 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-02T14:21:00,889 INFO [StoreOpener-9c93976ab5d5804c690c7e75c9981484-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:00,892 INFO [StoreOpener-9c93976ab5d5804c690c7e75c9981484-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:21:00,892 INFO [StoreOpener-9c93976ab5d5804c690c7e75c9981484-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c93976ab5d5804c690c7e75c9981484 columnFamilyName A 2024-12-02T14:21:00,892 DEBUG [StoreOpener-9c93976ab5d5804c690c7e75c9981484-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:00,893 INFO [StoreOpener-9c93976ab5d5804c690c7e75c9981484-1 {}] regionserver.HStore(327): Store=9c93976ab5d5804c690c7e75c9981484/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:21:00,893 INFO [StoreOpener-9c93976ab5d5804c690c7e75c9981484-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:00,894 INFO [StoreOpener-9c93976ab5d5804c690c7e75c9981484-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:21:00,894 INFO [StoreOpener-9c93976ab5d5804c690c7e75c9981484-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c93976ab5d5804c690c7e75c9981484 columnFamilyName B 2024-12-02T14:21:00,894 DEBUG [StoreOpener-9c93976ab5d5804c690c7e75c9981484-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:00,894 INFO [StoreOpener-9c93976ab5d5804c690c7e75c9981484-1 {}] regionserver.HStore(327): Store=9c93976ab5d5804c690c7e75c9981484/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:21:00,894 INFO [StoreOpener-9c93976ab5d5804c690c7e75c9981484-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:00,895 INFO [StoreOpener-9c93976ab5d5804c690c7e75c9981484-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:21:00,895 INFO [StoreOpener-9c93976ab5d5804c690c7e75c9981484-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c93976ab5d5804c690c7e75c9981484 columnFamilyName C 2024-12-02T14:21:00,895 DEBUG [StoreOpener-9c93976ab5d5804c690c7e75c9981484-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:00,896 INFO [StoreOpener-9c93976ab5d5804c690c7e75c9981484-1 {}] regionserver.HStore(327): Store=9c93976ab5d5804c690c7e75c9981484/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:21:00,896 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:00,896 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:00,896 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:00,897 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:00,897 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:00,897 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:00,898 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T14:21:00,899 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:00,900 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T14:21:00,900 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 9c93976ab5d5804c690c7e75c9981484; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64634014, jitterRate=-0.03687813878059387}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T14:21:00,900 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:00,901 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 9c93976ab5d5804c690c7e75c9981484: Running coprocessor pre-open hook at 1733149260887Writing region info on filesystem at 1733149260887Initializing all the Stores at 1733149260888 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149260888Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149260889 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149260889Cleaning up temporary data from old regions at 1733149260897 (+8 ms)Running coprocessor post-open hooks at 1733149260900 (+3 ms)Region opened successfully at 1733149260901 (+1 ms) 2024-12-02T14:21:00,901 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., pid=114, masterSystemTime=1733149260881 2024-12-02T14:21:00,903 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:00,903 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:00,903 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=9c93976ab5d5804c690c7e75c9981484, regionState=OPEN, openSeqNum=2, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:00,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:21:00,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-02T14:21:00,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 in 178 msec 2024-12-02T14:21:00,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-02T14:21:00,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c93976ab5d5804c690c7e75c9981484, ASSIGN in 336 msec 2024-12-02T14:21:00,908 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T14:21:00,908 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149260908"}]},"ts":"1733149260908"} 2024-12-02T14:21:00,909 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T14:21:00,910 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T14:21:00,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-12-02T14:21:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-02T14:21:01,898 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:01,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c7159d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:01,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:01,901 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:01,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:01,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:01,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fec6310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:01,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:01,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:01,903 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:01,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@258ff37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,904 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:01,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:01,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:01,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:01,908 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T14:21:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is a3a61c9ba14f,35877,1733149137941 2024-12-02T14:21:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@26dc739f 2024-12-02T14:21:01,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T14:21:01,910 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T14:21:01,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed28b71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:01,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:01,913 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:01,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:01,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:01,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ec8703, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:01,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:01,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:01,914 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b84232, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ded6cd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:01,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:01,916 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34cb429a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:01,917 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a3713f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27bfce74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,918 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:01,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:01,919 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be7d8cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:01,920 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:01,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45680853, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a52e343, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,922 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:01,922 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:01,923 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@317b35d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:01,924 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:01,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c11da82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@335cae3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:01,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:01,927 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bcee3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:01,928 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:01,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f4f1d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d6cc063, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,930 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:01,930 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:01,930 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e139208, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:01,932 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a0c6b3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bf63333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:01,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:01,934 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15c53b2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:01,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:01,936 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f350d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fd006e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:01,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:01,939 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe7567d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:01,941 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0fbd48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fddce7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,942 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:01,942 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:01,943 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:01,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:01,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:01,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1457f53c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:01,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:01,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:01,944 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:01,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@de3f9d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cec2e69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,945 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:01,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:01,946 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f3080a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:01,947 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1108379b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:01,948 DEBUG [Thread-1921 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:01,948 DEBUG [Thread-1927 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:01,948 DEBUG [Thread-1923 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:01,949 DEBUG [Thread-1930 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:01,949 DEBUG [Thread-1919 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:01,950 DEBUG [Thread-1934 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:01,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:01,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:01,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:01,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:01,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:01,950 DEBUG [Thread-1925 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:01,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:01,950 DEBUG [Thread-1932 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:01,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:01,950 DEBUG [Thread-1938 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:01,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:01,951 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:21:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:01,951 DEBUG [Thread-1936 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:01,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:01,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:01,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:01,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:01,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:01,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:01,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:01,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-02T14:21:01,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:01,954 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:01,954 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:01,954 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:21:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T14:21:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:21:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:21:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:21:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:21:01,956 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:21:01,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:21:01,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:21:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:21:01,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:01,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:01,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:21:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:21:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:21:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2] 2024-12-02T14:21:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:01,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T14:21:01,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:01,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:01,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:01,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:01,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:01,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:01,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149321985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:01,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:01,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:01,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/a4c3b68f1114435ba8b6c98a0541b374 is 50, key is test_row_1/A:col10/1733149261962/Put/seqid=0 2024-12-02T14:21:01,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:01,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149321987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:01,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:01,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:01,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:01,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149321990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:01,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:01,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149321992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149321991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742251_1427 (size=9657) 2024-12-02T14:21:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T14:21:02,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149322099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:02,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149322099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:02,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T14:21:02,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:02,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:02,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:02,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149322108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:02,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149322108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149322108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T14:21:02,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:02,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:02,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:02,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T14:21:02,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149322308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:02,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149322308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149322318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:02,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:02,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149322318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:02,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:02,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149322318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:02,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/a4c3b68f1114435ba8b6c98a0541b374 2024-12-02T14:21:02,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T14:21:02,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:02,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:02,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:02,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/9a52f4fbd8cf48e381d0f22738e6c9be is 50, key is test_row_1/B:col10/1733149261962/Put/seqid=0 2024-12-02T14:21:02,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742252_1428 (size=9657) 2024-12-02T14:21:02,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T14:21:02,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:02,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:02,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:02,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T14:21:02,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149322618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:02,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149322618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:02,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149322628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:02,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:02,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149322628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:02,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:02,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149322628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:02,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:02,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:02,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T14:21:02,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:02,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:02,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:02,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/9a52f4fbd8cf48e381d0f22738e6c9be 2024-12-02T14:21:02,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/14e573f7e4964c40a27df72513a39f80 is 50, key is test_row_1/C:col10/1733149261962/Put/seqid=0 2024-12-02T14:21:02,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742253_1429 (size=9657) 2024-12-02T14:21:02,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T14:21:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:02,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:03,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T14:21:03,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:03,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:03,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:03,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:03,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T14:21:03,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149323128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:03,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149323128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:03,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149323138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:03,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149323138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:03,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149323138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:03,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T14:21:03,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:03,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:03,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:03,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:03,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:03,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/14e573f7e4964c40a27df72513a39f80 2024-12-02T14:21:03,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/a4c3b68f1114435ba8b6c98a0541b374 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/a4c3b68f1114435ba8b6c98a0541b374 2024-12-02T14:21:03,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/a4c3b68f1114435ba8b6c98a0541b374, entries=100, sequenceid=13, filesize=9.4 K 2024-12-02T14:21:03,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/9a52f4fbd8cf48e381d0f22738e6c9be as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/9a52f4fbd8cf48e381d0f22738e6c9be 2024-12-02T14:21:03,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/9a52f4fbd8cf48e381d0f22738e6c9be, entries=100, sequenceid=13, filesize=9.4 K 2024-12-02T14:21:03,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/14e573f7e4964c40a27df72513a39f80 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/14e573f7e4964c40a27df72513a39f80 2024-12-02T14:21:03,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/14e573f7e4964c40a27df72513a39f80, entries=100, sequenceid=13, filesize=9.4 K 2024-12-02T14:21:03,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9c93976ab5d5804c690c7e75c9981484 in 1306ms, sequenceid=13, compaction requested=false 2024-12-02T14:21:03,273 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-02T14:21:03,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:03,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T14:21:03,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:03,331 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T14:21:03,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:03,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:03,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:03,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:03,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:03,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:03,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/8554089ac7f24b699af9e9bac04cacb0 is 50, key is test_row_0/A:col10/1733149261990/Put/seqid=0 2024-12-02T14:21:03,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742254_1430 (size=12001) 2024-12-02T14:21:03,739 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/8554089ac7f24b699af9e9bac04cacb0 2024-12-02T14:21:03,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/3ffbdd34cd82476c9b8f190d7d086255 is 50, key is test_row_0/B:col10/1733149261990/Put/seqid=0 2024-12-02T14:21:03,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742255_1431 (size=12001) 2024-12-02T14:21:04,076 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T14:21:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T14:21:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:04,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:04,148 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/3ffbdd34cd82476c9b8f190d7d086255 2024-12-02T14:21:04,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/51a9422b52dd41e2b5802783320367ad is 50, key is test_row_0/C:col10/1733149261990/Put/seqid=0 2024-12-02T14:21:04,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742256_1432 (size=12001) 2024-12-02T14:21:04,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149324167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149324167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149324168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149324169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149324169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149324278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149324279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149324279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149324279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149324279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149324489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149324489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149324489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149324489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149324489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,558 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/51a9422b52dd41e2b5802783320367ad 2024-12-02T14:21:04,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/8554089ac7f24b699af9e9bac04cacb0 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8554089ac7f24b699af9e9bac04cacb0 2024-12-02T14:21:04,564 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8554089ac7f24b699af9e9bac04cacb0, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T14:21:04,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/3ffbdd34cd82476c9b8f190d7d086255 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/3ffbdd34cd82476c9b8f190d7d086255 2024-12-02T14:21:04,567 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/3ffbdd34cd82476c9b8f190d7d086255, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T14:21:04,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/51a9422b52dd41e2b5802783320367ad as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/51a9422b52dd41e2b5802783320367ad 2024-12-02T14:21:04,571 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/51a9422b52dd41e2b5802783320367ad, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T14:21:04,571 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9c93976ab5d5804c690c7e75c9981484 in 1240ms, sequenceid=33, compaction requested=false 2024-12-02T14:21:04,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:04,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:04,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-02T14:21:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-02T14:21:04,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-02T14:21:04,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6170 sec 2024-12-02T14:21:04,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.6230 sec 2024-12-02T14:21:04,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:04,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T14:21:04,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:04,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:04,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:04,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:04,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:04,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:04,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/9ff48cde35ab4133a06b731b0789a6db is 50, key is test_row_0/A:col10/1733149264799/Put/seqid=0 2024-12-02T14:21:04,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742257_1433 (size=14341) 2024-12-02T14:21:04,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149324820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149324825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149324825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149324826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149324827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149324938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149324938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149324938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149324938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:04,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:04,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149324948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:04,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:04,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149325148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149325158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149325158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149325158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149325158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/9ff48cde35ab4133a06b731b0789a6db 2024-12-02T14:21:05,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/ab907ce7b2bd4ff1ae6d413d271dbc1d is 50, key is test_row_0/B:col10/1733149264799/Put/seqid=0 2024-12-02T14:21:05,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742258_1434 (size=12001) 2024-12-02T14:21:05,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/ab907ce7b2bd4ff1ae6d413d271dbc1d 2024-12-02T14:21:05,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/8856ab80844c47bb8afe91c92945b1f1 is 50, key is test_row_0/C:col10/1733149264799/Put/seqid=0 2024-12-02T14:21:05,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742259_1435 (size=12001) 2024-12-02T14:21:05,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/8856ab80844c47bb8afe91c92945b1f1 2024-12-02T14:21:05,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/9ff48cde35ab4133a06b731b0789a6db as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/9ff48cde35ab4133a06b731b0789a6db 2024-12-02T14:21:05,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/9ff48cde35ab4133a06b731b0789a6db, entries=200, sequenceid=46, filesize=14.0 K 2024-12-02T14:21:05,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/ab907ce7b2bd4ff1ae6d413d271dbc1d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/ab907ce7b2bd4ff1ae6d413d271dbc1d 2024-12-02T14:21:05,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/ab907ce7b2bd4ff1ae6d413d271dbc1d, entries=150, sequenceid=46, filesize=11.7 K 2024-12-02T14:21:05,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/8856ab80844c47bb8afe91c92945b1f1 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/8856ab80844c47bb8afe91c92945b1f1 2024-12-02T14:21:05,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/8856ab80844c47bb8afe91c92945b1f1, entries=150, sequenceid=46, filesize=11.7 K 2024-12-02T14:21:05,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9c93976ab5d5804c690c7e75c9981484 in 442ms, sequenceid=46, compaction requested=true 2024-12-02T14:21:05,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:05,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:21:05,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:05,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:21:05,243 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:05,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:05,243 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:05,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:21:05,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:05,243 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:05,243 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:05,243 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/B is initiating minor compaction (all files) 2024-12-02T14:21:05,243 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/A is initiating minor compaction (all files) 2024-12-02T14:21:05,244 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/A in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:05,244 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/B in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:05,244 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/9a52f4fbd8cf48e381d0f22738e6c9be, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/3ffbdd34cd82476c9b8f190d7d086255, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/ab907ce7b2bd4ff1ae6d413d271dbc1d] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=32.9 K 2024-12-02T14:21:05,244 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/a4c3b68f1114435ba8b6c98a0541b374, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8554089ac7f24b699af9e9bac04cacb0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/9ff48cde35ab4133a06b731b0789a6db] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=35.2 K 2024-12-02T14:21:05,244 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4c3b68f1114435ba8b6c98a0541b374, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733149261962 2024-12-02T14:21:05,244 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a52f4fbd8cf48e381d0f22738e6c9be, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733149261962 2024-12-02T14:21:05,244 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8554089ac7f24b699af9e9bac04cacb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733149261975 2024-12-02T14:21:05,244 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ffbdd34cd82476c9b8f190d7d086255, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733149261975 2024-12-02T14:21:05,245 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting ab907ce7b2bd4ff1ae6d413d271dbc1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733149264167 2024-12-02T14:21:05,245 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ff48cde35ab4133a06b731b0789a6db, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733149264166 2024-12-02T14:21:05,252 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#B#compaction#360 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:05,253 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/68d4bcf3f4844a8ebccb158c0e6695f7 is 50, key is test_row_0/B:col10/1733149264799/Put/seqid=0 2024-12-02T14:21:05,254 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#A#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:05,254 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/e764e2aa12dc49598e35239cd0a43b76 is 50, key is test_row_0/A:col10/1733149264799/Put/seqid=0 2024-12-02T14:21:05,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742260_1436 (size=12104) 2024-12-02T14:21:05,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742261_1437 (size=12104) 2024-12-02T14:21:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:05,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T14:21:05,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:05,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:05,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:05,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:05,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:05,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:05,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/120566003b46423d9a45e7edd65d9be3 is 50, key is test_row_0/A:col10/1733149264826/Put/seqid=0 2024-12-02T14:21:05,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742262_1438 (size=14341) 2024-12-02T14:21:05,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149325483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149325485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149325509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149325509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149325509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149325619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149325619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149325628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149325628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149325628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,662 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/68d4bcf3f4844a8ebccb158c0e6695f7 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/68d4bcf3f4844a8ebccb158c0e6695f7 2024-12-02T14:21:05,665 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/B of 9c93976ab5d5804c690c7e75c9981484 into 68d4bcf3f4844a8ebccb158c0e6695f7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:05,665 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:05,665 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/B, priority=13, startTime=1733149265243; duration=0sec 2024-12-02T14:21:05,665 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:05,665 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:B 2024-12-02T14:21:05,665 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:05,666 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:05,666 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/C is initiating minor compaction (all files) 2024-12-02T14:21:05,666 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/C in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:05,666 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/14e573f7e4964c40a27df72513a39f80, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/51a9422b52dd41e2b5802783320367ad, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/8856ab80844c47bb8afe91c92945b1f1] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=32.9 K 2024-12-02T14:21:05,666 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 14e573f7e4964c40a27df72513a39f80, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733149261962 2024-12-02T14:21:05,667 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 51a9422b52dd41e2b5802783320367ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733149261975 2024-12-02T14:21:05,667 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 8856ab80844c47bb8afe91c92945b1f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733149264167 2024-12-02T14:21:05,671 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/e764e2aa12dc49598e35239cd0a43b76 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/e764e2aa12dc49598e35239cd0a43b76 2024-12-02T14:21:05,673 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#C#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:05,674 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/d01003f9ca3f43b8aace459342aa70ad is 50, key is test_row_0/C:col10/1733149264799/Put/seqid=0 2024-12-02T14:21:05,674 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/A of 9c93976ab5d5804c690c7e75c9981484 into e764e2aa12dc49598e35239cd0a43b76(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:05,674 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:05,674 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/A, priority=13, startTime=1733149265242; duration=0sec 2024-12-02T14:21:05,675 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:05,675 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:A 2024-12-02T14:21:05,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742263_1439 (size=12104) 2024-12-02T14:21:05,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149325828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149325828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149325838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149325838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149325838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:05,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/120566003b46423d9a45e7edd65d9be3 2024-12-02T14:21:05,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/5835ed668c384b6fa0639dfe1feeaf81 is 50, key is test_row_0/B:col10/1733149264826/Put/seqid=0 2024-12-02T14:21:05,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742264_1440 (size=12001) 2024-12-02T14:21:06,080 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/d01003f9ca3f43b8aace459342aa70ad as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/d01003f9ca3f43b8aace459342aa70ad 2024-12-02T14:21:06,083 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/C of 9c93976ab5d5804c690c7e75c9981484 into d01003f9ca3f43b8aace459342aa70ad(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:06,083 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:06,083 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/C, priority=13, startTime=1733149265243; duration=0sec 2024-12-02T14:21:06,083 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:06,083 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:C 2024-12-02T14:21:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T14:21:06,118 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:06,119 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:21:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-02T14:21:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T14:21:06,120 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:21:06,121 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:21:06,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:21:06,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149326139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:06,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149326139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:06,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149326148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149326148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149326148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T14:21:06,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T14:21:06,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:06,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:06,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:06,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:06,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:06,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/5835ed668c384b6fa0639dfe1feeaf81 2024-12-02T14:21:06,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/26bcd0eede984095b705269218b58615 is 50, key is test_row_0/C:col10/1733149264826/Put/seqid=0 2024-12-02T14:21:06,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742265_1441 (size=12001) 2024-12-02T14:21:06,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T14:21:06,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:06,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:06,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:06,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:06,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T14:21:06,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T14:21:06,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:06,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:06,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:06,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:06,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:06,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149326648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:06,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:06,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149326657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:06,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:06,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149326658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149326659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:06,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149326659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:06,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/26bcd0eede984095b705269218b58615 2024-12-02T14:21:06,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/120566003b46423d9a45e7edd65d9be3 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/120566003b46423d9a45e7edd65d9be3 2024-12-02T14:21:06,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/120566003b46423d9a45e7edd65d9be3, entries=200, sequenceid=66, filesize=14.0 K 2024-12-02T14:21:06,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/5835ed668c384b6fa0639dfe1feeaf81 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/5835ed668c384b6fa0639dfe1feeaf81 2024-12-02T14:21:06,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/5835ed668c384b6fa0639dfe1feeaf81, entries=150, sequenceid=66, filesize=11.7 K 2024-12-02T14:21:06,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/26bcd0eede984095b705269218b58615 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/26bcd0eede984095b705269218b58615 2024-12-02T14:21:06,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/26bcd0eede984095b705269218b58615, entries=150, sequenceid=66, filesize=11.7 K 2024-12-02T14:21:06,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for 9c93976ab5d5804c690c7e75c9981484 in 1243ms, sequenceid=66, compaction requested=false 2024-12-02T14:21:06,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:06,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T14:21:06,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:06,732 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T14:21:06,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:06,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:06,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:06,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:06,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:06,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:06,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/c6aaa66236a24b869e1cdfba1ca31f1d is 50, key is test_row_0/A:col10/1733149265472/Put/seqid=0 2024-12-02T14:21:06,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742266_1442 (size=12001) 2024-12-02T14:21:06,746 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/c6aaa66236a24b869e1cdfba1ca31f1d 2024-12-02T14:21:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T14:21:06,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/593eb9e51cac446891d5ec6d0c135f54 is 50, key is test_row_0/B:col10/1733149265472/Put/seqid=0 2024-12-02T14:21:06,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742267_1443 (size=12001) 2024-12-02T14:21:07,173 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/593eb9e51cac446891d5ec6d0c135f54 2024-12-02T14:21:07,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/bfe245a1b9874e38beae805ef5f20786 is 50, key is test_row_0/C:col10/1733149265472/Put/seqid=0 2024-12-02T14:21:07,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742268_1444 (size=12001) 2024-12-02T14:21:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T14:21:07,581 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/bfe245a1b9874e38beae805ef5f20786 2024-12-02T14:21:07,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/c6aaa66236a24b869e1cdfba1ca31f1d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c6aaa66236a24b869e1cdfba1ca31f1d 2024-12-02T14:21:07,588 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c6aaa66236a24b869e1cdfba1ca31f1d, entries=150, sequenceid=83, filesize=11.7 K 2024-12-02T14:21:07,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/593eb9e51cac446891d5ec6d0c135f54 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/593eb9e51cac446891d5ec6d0c135f54 2024-12-02T14:21:07,591 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/593eb9e51cac446891d5ec6d0c135f54, entries=150, sequenceid=83, filesize=11.7 K 2024-12-02T14:21:07,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/bfe245a1b9874e38beae805ef5f20786 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/bfe245a1b9874e38beae805ef5f20786 2024-12-02T14:21:07,595 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/bfe245a1b9874e38beae805ef5f20786, entries=150, sequenceid=83, filesize=11.7 K 2024-12-02T14:21:07,596 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 9c93976ab5d5804c690c7e75c9981484 in 864ms, sequenceid=83, compaction requested=true 2024-12-02T14:21:07,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:07,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:07,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-02T14:21:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-02T14:21:07,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-02T14:21:07,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4760 sec 2024-12-02T14:21:07,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.4810 sec 2024-12-02T14:21:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:07,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T14:21:07,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:07,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:07,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:07,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:07,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:07,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:07,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/ae25e2bf90564aa6b25c22ff4e36bda0 is 50, key is test_row_0/A:col10/1733149267683/Put/seqid=0 2024-12-02T14:21:07,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742269_1445 (size=16681) 2024-12-02T14:21:07,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/ae25e2bf90564aa6b25c22ff4e36bda0 2024-12-02T14:21:07,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/07f38a6f44734db8a97c3194158e58e6 is 50, key is test_row_0/B:col10/1733149267683/Put/seqid=0 2024-12-02T14:21:07,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742270_1446 (size=12001) 2024-12-02T14:21:07,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149267713, max timestamp=1733149267726Num uniques 859; 2024-12-02T14:21:07,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:21:07,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149267713, max timestamp=1733149267726Num uniques 859; 2024-12-02T14:21:07,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:21:07,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149267713, max timestamp=1733149267726Num uniques 859; 2024-12-02T14:21:07,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:21:07,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149327749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:07,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149327750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:07,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:07,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149327754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:07,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:07,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149327760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:07,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149327761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:07,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:07,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149327868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:07,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149327869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:07,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149327869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:07,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149327878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:07,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149327878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:07,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:08,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149328089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:08,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149328089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149328089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149328089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:08,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149328098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:08,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/07f38a6f44734db8a97c3194158e58e6 2024-12-02T14:21:08,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/d9597c9c3da64f3085c2b16423024199 is 50, key is test_row_0/C:col10/1733149267683/Put/seqid=0 2024-12-02T14:21:08,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742271_1447 (size=12001) 2024-12-02T14:21:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T14:21:08,269 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:08,269 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:21:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-02T14:21:08,270 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:21:08,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T14:21:08,271 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:21:08,271 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:21:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T14:21:08,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149328399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:08,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149328399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:08,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:08,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:08,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149328408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:08,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149328408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:08,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-02T14:21:08,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:08,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:08,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:08,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:08,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:08,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149328418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:08,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/d9597c9c3da64f3085c2b16423024199 2024-12-02T14:21:08,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/ae25e2bf90564aa6b25c22ff4e36bda0 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/ae25e2bf90564aa6b25c22ff4e36bda0 2024-12-02T14:21:08,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/ae25e2bf90564aa6b25c22ff4e36bda0, entries=250, sequenceid=94, filesize=16.3 K 2024-12-02T14:21:08,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/07f38a6f44734db8a97c3194158e58e6 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/07f38a6f44734db8a97c3194158e58e6 2024-12-02T14:21:08,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/07f38a6f44734db8a97c3194158e58e6, entries=150, sequenceid=94, filesize=11.7 K 2024-12-02T14:21:08,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/d9597c9c3da64f3085c2b16423024199 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/d9597c9c3da64f3085c2b16423024199 2024-12-02T14:21:08,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/d9597c9c3da64f3085c2b16423024199, entries=150, sequenceid=94, filesize=11.7 K 2024-12-02T14:21:08,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 9c93976ab5d5804c690c7e75c9981484 in 850ms, sequenceid=94, compaction requested=true 2024-12-02T14:21:08,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:08,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:21:08,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:08,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:21:08,533 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:21:08,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:08,533 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:21:08,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:21:08,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:08,534 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:21:08,534 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/B is initiating minor compaction (all files) 2024-12-02T14:21:08,535 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/B in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:08,535 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/68d4bcf3f4844a8ebccb158c0e6695f7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/5835ed668c384b6fa0639dfe1feeaf81, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/593eb9e51cac446891d5ec6d0c135f54, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/07f38a6f44734db8a97c3194158e58e6] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=47.0 K 2024-12-02T14:21:08,535 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55127 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:21:08,535 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/A is initiating minor compaction (all files) 2024-12-02T14:21:08,535 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/A in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:08,535 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/e764e2aa12dc49598e35239cd0a43b76, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/120566003b46423d9a45e7edd65d9be3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c6aaa66236a24b869e1cdfba1ca31f1d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/ae25e2bf90564aa6b25c22ff4e36bda0] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=53.8 K 2024-12-02T14:21:08,535 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 68d4bcf3f4844a8ebccb158c0e6695f7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733149264167 2024-12-02T14:21:08,535 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting e764e2aa12dc49598e35239cd0a43b76, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733149264167 2024-12-02T14:21:08,535 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 5835ed668c384b6fa0639dfe1feeaf81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733149264823 2024-12-02T14:21:08,536 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 120566003b46423d9a45e7edd65d9be3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733149264823 2024-12-02T14:21:08,536 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 593eb9e51cac446891d5ec6d0c135f54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733149265472 2024-12-02T14:21:08,536 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6aaa66236a24b869e1cdfba1ca31f1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733149265472 2024-12-02T14:21:08,536 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 07f38a6f44734db8a97c3194158e58e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733149267669 2024-12-02T14:21:08,536 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae25e2bf90564aa6b25c22ff4e36bda0, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733149267669 2024-12-02T14:21:08,543 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#A#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:08,543 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/424afcdfa49a41f7bb32f917b39805de is 50, key is test_row_0/A:col10/1733149267683/Put/seqid=0 2024-12-02T14:21:08,543 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#B#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:08,544 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/05a14d855ab844cea9eccbba0eb9b7c0 is 50, key is test_row_0/B:col10/1733149267683/Put/seqid=0 2024-12-02T14:21:08,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742272_1448 (size=12241) 2024-12-02T14:21:08,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742273_1449 (size=12241) 2024-12-02T14:21:08,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-02T14:21:08,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:08,575 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-02T14:21:08,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:08,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:21:08,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:08,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:21:08,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:08,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:21:08,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/9b609ba1c48c4940a6817b62a83b7529 is 50, key is test_row_0/A:col10/1733149267748/Put/seqid=0 2024-12-02T14:21:08,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742274_1450 (size=12001) 2024-12-02T14:21:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T14:21:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T14:21:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:08,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:08,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149328939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:08,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149328940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:08,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149328941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:08,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149328941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:08,958 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/424afcdfa49a41f7bb32f917b39805de as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/424afcdfa49a41f7bb32f917b39805de 2024-12-02T14:21:08,959 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/05a14d855ab844cea9eccbba0eb9b7c0 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/05a14d855ab844cea9eccbba0eb9b7c0 2024-12-02T14:21:08,962 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/B of 9c93976ab5d5804c690c7e75c9981484 into 05a14d855ab844cea9eccbba0eb9b7c0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:08,962 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:08,962 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/B, priority=12, startTime=1733149268533; duration=0sec 2024-12-02T14:21:08,963 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:08,963 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:B 2024-12-02T14:21:08,963 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:21:08,964 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/A of 9c93976ab5d5804c690c7e75c9981484 into 424afcdfa49a41f7bb32f917b39805de(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:08,964 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:08,965 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/A, priority=12, startTime=1733149268533; duration=0sec 2024-12-02T14:21:08,965 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:08,965 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:A 2024-12-02T14:21:08,965 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:21:08,965 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/C is initiating minor compaction (all files) 2024-12-02T14:21:08,965 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/C in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:08,965 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/d01003f9ca3f43b8aace459342aa70ad, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/26bcd0eede984095b705269218b58615, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/bfe245a1b9874e38beae805ef5f20786, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/d9597c9c3da64f3085c2b16423024199] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=47.0 K 2024-12-02T14:21:08,966 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting d01003f9ca3f43b8aace459342aa70ad, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733149264167 2024-12-02T14:21:08,966 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 26bcd0eede984095b705269218b58615, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733149264823 2024-12-02T14:21:08,966 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting bfe245a1b9874e38beae805ef5f20786, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733149265472 2024-12-02T14:21:08,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,966 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting d9597c9c3da64f3085c2b16423024199, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733149267669 2024-12-02T14:21:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149328952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:08,974 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#C#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:08,974 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/98019000ff9742dda372f0a34b56c872 is 50, key is test_row_0/C:col10/1733149267683/Put/seqid=0 2024-12-02T14:21:08,982 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/9b609ba1c48c4940a6817b62a83b7529 2024-12-02T14:21:08,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742275_1451 (size=12241) 2024-12-02T14:21:08,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/e007275ab6884d64b71f0aa4d967e3dd is 50, key is test_row_0/B:col10/1733149267748/Put/seqid=0 2024-12-02T14:21:08,993 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/98019000ff9742dda372f0a34b56c872 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/98019000ff9742dda372f0a34b56c872 2024-12-02T14:21:08,997 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/C of 9c93976ab5d5804c690c7e75c9981484 into 98019000ff9742dda372f0a34b56c872(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:08,997 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:08,997 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/C, priority=12, startTime=1733149268533; duration=0sec 2024-12-02T14:21:08,997 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:08,997 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:C 2024-12-02T14:21:09,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742276_1452 (size=12001) 2024-12-02T14:21:09,006 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/e007275ab6884d64b71f0aa4d967e3dd 2024-12-02T14:21:09,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/7032af9358594d3b917dc1b77774df69 is 50, key is test_row_0/C:col10/1733149267748/Put/seqid=0 2024-12-02T14:21:09,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742277_1453 (size=12001) 2024-12-02T14:21:09,016 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/7032af9358594d3b917dc1b77774df69 2024-12-02T14:21:09,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/9b609ba1c48c4940a6817b62a83b7529 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/9b609ba1c48c4940a6817b62a83b7529 2024-12-02T14:21:09,023 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/9b609ba1c48c4940a6817b62a83b7529, entries=150, sequenceid=129, filesize=11.7 K 2024-12-02T14:21:09,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/e007275ab6884d64b71f0aa4d967e3dd as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/e007275ab6884d64b71f0aa4d967e3dd 2024-12-02T14:21:09,031 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/e007275ab6884d64b71f0aa4d967e3dd, entries=150, sequenceid=129, filesize=11.7 K 2024-12-02T14:21:09,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/7032af9358594d3b917dc1b77774df69 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/7032af9358594d3b917dc1b77774df69 2024-12-02T14:21:09,034 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/7032af9358594d3b917dc1b77774df69, entries=150, sequenceid=129, filesize=11.7 K 2024-12-02T14:21:09,036 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 9c93976ab5d5804c690c7e75c9981484 in 461ms, sequenceid=129, compaction requested=false 2024-12-02T14:21:09,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:09,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:09,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-02T14:21:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-02T14:21:09,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-02T14:21:09,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 769 msec 2024-12-02T14:21:09,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 775 msec 2024-12-02T14:21:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:09,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T14:21:09,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:09,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:09,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:09,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:09,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:09,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:09,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/d78001eeea2140d79fb0b2bc3d69b053 is 50, key is test_row_0/A:col10/1733149269061/Put/seqid=0 2024-12-02T14:21:09,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742278_1454 (size=12151) 2024-12-02T14:21:09,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149329103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:09,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149329108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:09,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149329109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:09,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149329110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:09,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:09,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:09,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149329218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:09,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:09,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149329229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149329229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149329229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T14:21:09,409 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:09,409 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:21:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-02T14:21:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T14:21:09,410 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:21:09,411 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:21:09,411 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:21:09,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149329429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:09,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149329439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:09,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149329439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:09,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:09,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149329439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:09,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:09,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:09,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/d78001eeea2140d79fb0b2bc3d69b053 2024-12-02T14:21:09,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/04ac8374a0944755a408e003e57870b7 is 50, key is test_row_0/B:col10/1733149269061/Put/seqid=0 2024-12-02T14:21:09,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742279_1455 (size=12151) 2024-12-02T14:21:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T14:21:09,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T14:21:09,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:09,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:09,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:09,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:09,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:09,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T14:21:09,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:09,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:09,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:09,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:09,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T14:21:09,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149329738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:09,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149329749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:09,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149329749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:09,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149329749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:09,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:09,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:09,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T14:21:09,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:09,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:09,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:09,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:09,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:09,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/04ac8374a0944755a408e003e57870b7 2024-12-02T14:21:09,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/dd47c9bfbf094a2b958ee2a7922849f1 is 50, key is test_row_0/C:col10/1733149269061/Put/seqid=0 2024-12-02T14:21:09,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742280_1456 (size=12151) 2024-12-02T14:21:09,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/dd47c9bfbf094a2b958ee2a7922849f1 2024-12-02T14:21:09,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/d78001eeea2140d79fb0b2bc3d69b053 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/d78001eeea2140d79fb0b2bc3d69b053 2024-12-02T14:21:09,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/d78001eeea2140d79fb0b2bc3d69b053, entries=150, sequenceid=144, filesize=11.9 K 2024-12-02T14:21:09,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/04ac8374a0944755a408e003e57870b7 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/04ac8374a0944755a408e003e57870b7 2024-12-02T14:21:09,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/04ac8374a0944755a408e003e57870b7, entries=150, sequenceid=144, filesize=11.9 K 2024-12-02T14:21:09,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/dd47c9bfbf094a2b958ee2a7922849f1 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/dd47c9bfbf094a2b958ee2a7922849f1 2024-12-02T14:21:09,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/dd47c9bfbf094a2b958ee2a7922849f1, entries=150, sequenceid=144, filesize=11.9 K 2024-12-02T14:21:09,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9c93976ab5d5804c690c7e75c9981484 in 902ms, sequenceid=144, compaction requested=true 2024-12-02T14:21:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:21:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:09,966 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:09,966 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:21:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:09,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:21:09,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:09,968 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:09,968 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:09,968 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/B is initiating minor compaction (all files) 2024-12-02T14:21:09,968 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/A is initiating minor compaction (all files) 2024-12-02T14:21:09,968 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/B in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:09,968 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/A in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:09,968 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/424afcdfa49a41f7bb32f917b39805de, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/9b609ba1c48c4940a6817b62a83b7529, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/d78001eeea2140d79fb0b2bc3d69b053] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=35.5 K 2024-12-02T14:21:09,968 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/05a14d855ab844cea9eccbba0eb9b7c0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/e007275ab6884d64b71f0aa4d967e3dd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/04ac8374a0944755a408e003e57870b7] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=35.5 K 2024-12-02T14:21:09,968 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 424afcdfa49a41f7bb32f917b39805de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733149267669 2024-12-02T14:21:09,968 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 05a14d855ab844cea9eccbba0eb9b7c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733149267669 2024-12-02T14:21:09,969 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b609ba1c48c4940a6817b62a83b7529, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733149267748 2024-12-02T14:21:09,969 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting e007275ab6884d64b71f0aa4d967e3dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733149267748 2024-12-02T14:21:09,970 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting d78001eeea2140d79fb0b2bc3d69b053, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733149268940 2024-12-02T14:21:09,972 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 04ac8374a0944755a408e003e57870b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733149268940 2024-12-02T14:21:09,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149269097, max timestamp=1733149269975Num uniques 859; 2024-12-02T14:21:09,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:21:09,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149269097, max timestamp=1733149269975Num uniques 859; 2024-12-02T14:21:09,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:21:09,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149269097, max timestamp=1733149269975Num uniques 859; 2024-12-02T14:21:09,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:21:09,979 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#A#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:09,979 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/4ff6c2c4f18247b7bb19b874e29f1edd is 50, key is test_row_0/A:col10/1733149269061/Put/seqid=0 2024-12-02T14:21:09,982 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#B#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:09,982 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/6cacf0176b6c452c805db0f00353ecda is 50, key is test_row_0/B:col10/1733149269061/Put/seqid=0 2024-12-02T14:21:10,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T14:21:10,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742281_1457 (size=12493) 2024-12-02T14:21:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:10,022 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=46.96 KB heapSize=126 KB 2024-12-02T14:21:10,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742282_1458 (size=12493) 2024-12-02T14:21:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:21:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:21:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:21:10,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:10,032 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/4ff6c2c4f18247b7bb19b874e29f1edd as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/4ff6c2c4f18247b7bb19b874e29f1edd 2024-12-02T14:21:10,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/3bd2ba5962d240e0abbf12110db75987 is 50, key is test_row_0/A:col10/1733149269992/Put/seqid=0 2024-12-02T14:21:10,034 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/6cacf0176b6c452c805db0f00353ecda as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/6cacf0176b6c452c805db0f00353ecda 2024-12-02T14:21:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T14:21:10,045 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/B of 9c93976ab5d5804c690c7e75c9981484 into 6cacf0176b6c452c805db0f00353ecda(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:10,045 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/A of 9c93976ab5d5804c690c7e75c9981484 into 4ff6c2c4f18247b7bb19b874e29f1edd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:10,045 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:10,045 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:10,046 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/B, priority=13, startTime=1733149269966; duration=0sec 2024-12-02T14:21:10,046 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/A, priority=13, startTime=1733149269966; duration=0sec 2024-12-02T14:21:10,046 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:10,046 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:10,046 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:B 2024-12-02T14:21:10,046 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:A 2024-12-02T14:21:10,046 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:10,046 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:10,047 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/C is initiating minor compaction (all files) 2024-12-02T14:21:10,047 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/C in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:10,047 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/98019000ff9742dda372f0a34b56c872, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/7032af9358594d3b917dc1b77774df69, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/dd47c9bfbf094a2b958ee2a7922849f1] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=35.5 K 2024-12-02T14:21:10,047 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98019000ff9742dda372f0a34b56c872, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733149267669 2024-12-02T14:21:10,047 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7032af9358594d3b917dc1b77774df69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733149267748 2024-12-02T14:21:10,047 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd47c9bfbf094a2b958ee2a7922849f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733149268940 2024-12-02T14:21:10,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742283_1459 (size=12151) 2024-12-02T14:21:10,051 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/3bd2ba5962d240e0abbf12110db75987 2024-12-02T14:21:10,058 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#C#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:10,059 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/a799e6e8c1aa45058233bce73a049505 is 50, key is test_row_0/C:col10/1733149269061/Put/seqid=0 2024-12-02T14:21:10,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/71ceba4c482b458888672ee9ab78f48d is 50, key is test_row_0/B:col10/1733149269992/Put/seqid=0 2024-12-02T14:21:10,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742284_1460 (size=12493) 2024-12-02T14:21:10,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742285_1461 (size=12151) 2024-12-02T14:21:10,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149270048, max timestamp=1733149270170Num uniques 859; 2024-12-02T14:21:10,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149270048, max timestamp=1733149270170Num uniques 859; 2024-12-02T14:21:10,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:21:10,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:21:10,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149270048, max timestamp=1733149270170Num uniques 859; 2024-12-02T14:21:10,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:21:10,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149330272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149330273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149330274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149330275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149330277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149330388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149330388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149330388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149330398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149330398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,470 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/a799e6e8c1aa45058233bce73a049505 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/a799e6e8c1aa45058233bce73a049505 2024-12-02T14:21:10,474 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/C of 9c93976ab5d5804c690c7e75c9981484 into a799e6e8c1aa45058233bce73a049505(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:10,474 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:10,474 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/C, priority=13, startTime=1733149269967; duration=0sec 2024-12-02T14:21:10,474 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:10,474 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:C 2024-12-02T14:21:10,478 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/71ceba4c482b458888672ee9ab78f48d 2024-12-02T14:21:10,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/96b654d2fc594111a874410b90bda2d5 is 50, key is test_row_0/C:col10/1733149269992/Put/seqid=0 2024-12-02T14:21:10,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742286_1462 (size=12151) 2024-12-02T14:21:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T14:21:10,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149330598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149330598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149330598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149330609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149330609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,893 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/96b654d2fc594111a874410b90bda2d5 2024-12-02T14:21:10,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/3bd2ba5962d240e0abbf12110db75987 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/3bd2ba5962d240e0abbf12110db75987 2024-12-02T14:21:10,905 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/3bd2ba5962d240e0abbf12110db75987, entries=150, sequenceid=168, filesize=11.9 K 2024-12-02T14:21:10,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/71ceba4c482b458888672ee9ab78f48d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/71ceba4c482b458888672ee9ab78f48d 2024-12-02T14:21:10,910 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/71ceba4c482b458888672ee9ab78f48d, entries=150, sequenceid=168, filesize=11.9 K 2024-12-02T14:21:10,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/96b654d2fc594111a874410b90bda2d5 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/96b654d2fc594111a874410b90bda2d5 2024-12-02T14:21:10,915 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/96b654d2fc594111a874410b90bda2d5, entries=150, sequenceid=168, filesize=11.9 K 2024-12-02T14:21:10,916 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~125.95 KB/128976, currentSize=140.89 KB/144270 for 9c93976ab5d5804c690c7e75c9981484 in 894ms, sequenceid=168, compaction requested=false 2024-12-02T14:21:10,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:10,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:10,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-02T14:21:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-02T14:21:10,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:10,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-12-02T14:21:10,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:10,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T14:21:10,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:10,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T14:21:10,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:10,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T14:21:10,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-02T14:21:10,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5060 sec 2024-12-02T14:21:10,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.5130 sec 2024-12-02T14:21:10,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/1e5752a6081f45e7bf83fd051f2c5090 is 50, key is test_row_0/A:col10/1733149270271/Put/seqid=0 2024-12-02T14:21:10,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149330923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149330923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,932 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149330924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149330927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742287_1463 (size=12151) 2024-12-02T14:21:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:10,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149330931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:11,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149331038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:11,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149331038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:11,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149331039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:11,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149331049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:11,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149331049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:11,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149331259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:11,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149331259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:11,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149331259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149331259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149331259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:11,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/1e5752a6081f45e7bf83fd051f2c5090 2024-12-02T14:21:11,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/67e5f2f264af4f1da23c9fbde146cd06 is 50, key is test_row_0/B:col10/1733149270271/Put/seqid=0 2024-12-02T14:21:11,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742288_1464 (size=12151) 2024-12-02T14:21:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T14:21:11,558 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:11,559 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:21:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-02T14:21:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T14:21:11,560 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:21:11,561 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:21:11,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:21:11,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149331578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:11,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149331579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:11,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149331579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:11,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149331579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149331579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:11,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T14:21:11,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-02T14:21:11,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:11,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:11,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:11,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:11,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:11,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/67e5f2f264af4f1da23c9fbde146cd06 2024-12-02T14:21:11,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/1bfec387cadf4f64b60e0f024cad75d9 is 50, key is test_row_0/C:col10/1733149270271/Put/seqid=0 2024-12-02T14:21:11,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742289_1465 (size=12151) 2024-12-02T14:21:11,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-02T14:21:11,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:11,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:11,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:11,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:11,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T14:21:12,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-02T14:21:12,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:12,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:12,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:12,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:12,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:12,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149332089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:12,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:12,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:12,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:12,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149332098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:12,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149332098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:12,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149332098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:12,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149332098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:12,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/1bfec387cadf4f64b60e0f024cad75d9 2024-12-02T14:21:12,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/1e5752a6081f45e7bf83fd051f2c5090 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/1e5752a6081f45e7bf83fd051f2c5090 2024-12-02T14:21:12,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/1e5752a6081f45e7bf83fd051f2c5090, entries=150, sequenceid=209, filesize=11.9 K 2024-12-02T14:21:12,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/67e5f2f264af4f1da23c9fbde146cd06 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/67e5f2f264af4f1da23c9fbde146cd06 2024-12-02T14:21:12,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-02T14:21:12,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:12,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:12,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:12,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:12,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:12,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/67e5f2f264af4f1da23c9fbde146cd06, entries=150, sequenceid=209, filesize=11.9 K 2024-12-02T14:21:12,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/1bfec387cadf4f64b60e0f024cad75d9 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/1bfec387cadf4f64b60e0f024cad75d9 2024-12-02T14:21:12,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/1bfec387cadf4f64b60e0f024cad75d9, entries=150, sequenceid=209, filesize=11.9 K 2024-12-02T14:21:12,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 9c93976ab5d5804c690c7e75c9981484 in 1262ms, sequenceid=209, compaction requested=true 2024-12-02T14:21:12,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:12,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:21:12,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:12,182 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:12,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:21:12,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:12,182 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:12,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:21:12,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:12,183 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:12,183 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:12,183 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/B is initiating minor compaction (all files) 2024-12-02T14:21:12,183 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/A is initiating minor compaction (all files) 2024-12-02T14:21:12,183 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/B in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:12,183 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/A in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:12,183 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/4ff6c2c4f18247b7bb19b874e29f1edd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/3bd2ba5962d240e0abbf12110db75987, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/1e5752a6081f45e7bf83fd051f2c5090] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=35.9 K 2024-12-02T14:21:12,183 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/6cacf0176b6c452c805db0f00353ecda, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/71ceba4c482b458888672ee9ab78f48d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/67e5f2f264af4f1da23c9fbde146cd06] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=35.9 K 2024-12-02T14:21:12,183 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ff6c2c4f18247b7bb19b874e29f1edd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733149268940 2024-12-02T14:21:12,183 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cacf0176b6c452c805db0f00353ecda, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733149268940 2024-12-02T14:21:12,183 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bd2ba5962d240e0abbf12110db75987, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733149269988 2024-12-02T14:21:12,183 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 71ceba4c482b458888672ee9ab78f48d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733149269988 2024-12-02T14:21:12,184 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 67e5f2f264af4f1da23c9fbde146cd06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733149270233 2024-12-02T14:21:12,184 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e5752a6081f45e7bf83fd051f2c5090, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733149270233 2024-12-02T14:21:12,189 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#B#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:12,189 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/0711e73ab342430aa723494805e245da is 50, key is test_row_0/B:col10/1733149270271/Put/seqid=0 2024-12-02T14:21:12,190 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#A#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T14:21:12,190 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/0244e0e233524b9a90ecc236baaca543 is 50, key is test_row_0/A:col10/1733149270271/Put/seqid=0 2024-12-02T14:21:12,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742290_1466 (size=12595) 2024-12-02T14:21:12,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742291_1467 (size=12595) 2024-12-02T14:21:12,201 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/0244e0e233524b9a90ecc236baaca543 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/0244e0e233524b9a90ecc236baaca543 2024-12-02T14:21:12,204 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/A of 9c93976ab5d5804c690c7e75c9981484 into 0244e0e233524b9a90ecc236baaca543(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:12,204 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:12,204 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/A, priority=13, startTime=1733149272182; duration=0sec 2024-12-02T14:21:12,204 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:12,204 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:A 2024-12-02T14:21:12,205 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:12,205 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:12,205 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/C is initiating minor compaction (all files) 2024-12-02T14:21:12,205 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/C in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:12,205 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/a799e6e8c1aa45058233bce73a049505, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/96b654d2fc594111a874410b90bda2d5, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/1bfec387cadf4f64b60e0f024cad75d9] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=35.9 K 2024-12-02T14:21:12,206 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting a799e6e8c1aa45058233bce73a049505, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733149268940 2024-12-02T14:21:12,206 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96b654d2fc594111a874410b90bda2d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733149269988 2024-12-02T14:21:12,206 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bfec387cadf4f64b60e0f024cad75d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733149270233 2024-12-02T14:21:12,212 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#C#compaction#392 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:12,213 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/89cd22ab716641538ca1790d6d839f4a is 50, key is test_row_0/C:col10/1733149270271/Put/seqid=0 2024-12-02T14:21:12,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742292_1468 (size=12595) 2024-12-02T14:21:12,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-02T14:21:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:12,323 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T14:21:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:12,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/c38e5f528f564213b081ca9bc77c03b8 is 50, key is test_row_0/A:col10/1733149270929/Put/seqid=0 2024-12-02T14:21:12,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742293_1469 (size=12151) 2024-12-02T14:21:12,598 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/0711e73ab342430aa723494805e245da as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/0711e73ab342430aa723494805e245da 2024-12-02T14:21:12,602 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/B of 9c93976ab5d5804c690c7e75c9981484 into 0711e73ab342430aa723494805e245da(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:12,602 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:12,602 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/B, priority=13, startTime=1733149272182; duration=0sec 2024-12-02T14:21:12,602 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:12,602 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:B 2024-12-02T14:21:12,626 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/89cd22ab716641538ca1790d6d839f4a as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/89cd22ab716641538ca1790d6d839f4a 2024-12-02T14:21:12,630 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/C of 9c93976ab5d5804c690c7e75c9981484 into 89cd22ab716641538ca1790d6d839f4a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:12,630 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:12,630 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/C, priority=13, startTime=1733149272182; duration=0sec 2024-12-02T14:21:12,630 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:12,630 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:C 2024-12-02T14:21:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T14:21:12,731 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/c38e5f528f564213b081ca9bc77c03b8 2024-12-02T14:21:12,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/81f6363da23941c4b2795dc6551df979 is 50, key is test_row_0/B:col10/1733149270929/Put/seqid=0 2024-12-02T14:21:12,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742294_1470 (size=12151) 2024-12-02T14:21:13,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:13,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:13,139 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/81f6363da23941c4b2795dc6551df979 2024-12-02T14:21:13,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733149273131, max timestamp=1733149273141Num uniques 859; 2024-12-02T14:21:13,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733149273131, max timestamp=1733149273141Num uniques 859; 2024-12-02T14:21:13,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:21:13,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:21:13,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733149273131, max timestamp=1733149273141Num uniques 859; 2024-12-02T14:21:13,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:21:13,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/5a37891eac72493a9538cb59460aaef9 is 50, key is test_row_0/C:col10/1733149270929/Put/seqid=0 2024-12-02T14:21:13,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742295_1471 (size=12151) 2024-12-02T14:21:13,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149333170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149333171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733149273160, max timestamp=1733149273179Num uniques 1068; 2024-12-02T14:21:13,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733149273160, max timestamp=1733149273179Num uniques 1068; 2024-12-02T14:21:13,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-02T14:21:13,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-02T14:21:13,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733149273160, max timestamp=1733149273179Num uniques 1068; 2024-12-02T14:21:13,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-02T14:21:13,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149333221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149333232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149333233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149333288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149333288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149333339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149333348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149333348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149333498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149333498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149333549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,558 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/5a37891eac72493a9538cb59460aaef9 2024-12-02T14:21:13,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/c38e5f528f564213b081ca9bc77c03b8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c38e5f528f564213b081ca9bc77c03b8 2024-12-02T14:21:13,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149333558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,565 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c38e5f528f564213b081ca9bc77c03b8, entries=150, sequenceid=218, filesize=11.9 K 2024-12-02T14:21:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149333558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/81f6363da23941c4b2795dc6551df979 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/81f6363da23941c4b2795dc6551df979 2024-12-02T14:21:13,579 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/81f6363da23941c4b2795dc6551df979, entries=150, sequenceid=218, filesize=11.9 K 2024-12-02T14:21:13,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/5a37891eac72493a9538cb59460aaef9 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/5a37891eac72493a9538cb59460aaef9 2024-12-02T14:21:13,582 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/5a37891eac72493a9538cb59460aaef9, entries=150, sequenceid=218, filesize=11.9 K 2024-12-02T14:21:13,583 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 9c93976ab5d5804c690c7e75c9981484 in 1260ms, sequenceid=218, compaction requested=false 2024-12-02T14:21:13,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:13,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:13,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-02T14:21:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-02T14:21:13,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-02T14:21:13,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0230 sec 2024-12-02T14:21:13,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.0270 sec 2024-12-02T14:21:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T14:21:13,708 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:13,709 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:21:13,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-02T14:21:13,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T14:21:13,711 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:21:13,711 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:21:13,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:21:13,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:13,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=154.31 KB heapSize=435.23 KB 2024-12-02T14:21:13,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:13,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T14:21:13,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:13,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T14:21:13,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:13,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T14:21:13,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T14:21:13,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/8862fd315a5c45bbb89ef4f90d36998c is 50, key is test_row_0/A:col10/1733149273814/Put/seqid=0 2024-12-02T14:21:13,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742296_1472 (size=14741) 2024-12-02T14:21:13,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149333840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149333840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T14:21:13,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:13,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:13,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:13,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:13,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149333858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149333878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149333878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149333948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149333948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:14,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T14:21:14,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:14,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:14,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:14,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T14:21:14,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149334159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:14,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149334159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:14,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T14:21:14,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:14,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:14,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:14,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/8862fd315a5c45bbb89ef4f90d36998c 2024-12-02T14:21:14,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/5c5435d4f66f4df785fb17109cd9aa18 is 50, key is test_row_0/B:col10/1733149273814/Put/seqid=0 2024-12-02T14:21:14,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742297_1473 (size=12301) 2024-12-02T14:21:14,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T14:21:14,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:14,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:14,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:14,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T14:21:14,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149334378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:14,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:14,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149334389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:14,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149334389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:14,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:14,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:14,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T14:21:14,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:14,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:14,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:14,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149334468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:14,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:14,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149334479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:14,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T14:21:14,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:14,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:14,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:14,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/5c5435d4f66f4df785fb17109cd9aa18 2024-12-02T14:21:14,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/4ccb00362b804ea7b7f0f105efd68873 is 50, key is test_row_0/C:col10/1733149273814/Put/seqid=0 2024-12-02T14:21:14,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742298_1474 (size=12301) 2024-12-02T14:21:14,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T14:21:14,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:14,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:14,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:14,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T14:21:14,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T14:21:14,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:14,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:14,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:14,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149334988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:14,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149334988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:15,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/4ccb00362b804ea7b7f0f105efd68873 2024-12-02T14:21:15,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/8862fd315a5c45bbb89ef4f90d36998c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8862fd315a5c45bbb89ef4f90d36998c 2024-12-02T14:21:15,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8862fd315a5c45bbb89ef4f90d36998c, entries=200, sequenceid=275, filesize=14.4 K 2024-12-02T14:21:15,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/5c5435d4f66f4df785fb17109cd9aa18 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/5c5435d4f66f4df785fb17109cd9aa18 2024-12-02T14:21:15,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/5c5435d4f66f4df785fb17109cd9aa18, entries=150, sequenceid=275, filesize=12.0 K 2024-12-02T14:21:15,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/4ccb00362b804ea7b7f0f105efd68873 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/4ccb00362b804ea7b7f0f105efd68873 2024-12-02T14:21:15,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/4ccb00362b804ea7b7f0f105efd68873, entries=150, sequenceid=275, filesize=12.0 K 2024-12-02T14:21:15,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~435.19 KB/445632, currentSize=33.54 KB/34350 for 9c93976ab5d5804c690c7e75c9981484 in 1247ms, sequenceid=275, compaction requested=true 2024-12-02T14:21:15,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:15,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:21:15,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:15,063 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:15,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:21:15,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:15,063 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:15,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:21:15,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:15,063 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:15,063 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39487 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:15,064 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/A is initiating minor compaction (all files) 2024-12-02T14:21:15,064 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/B is initiating minor compaction (all files) 2024-12-02T14:21:15,064 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/B in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:15,064 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/A in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:15,064 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/0711e73ab342430aa723494805e245da, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/81f6363da23941c4b2795dc6551df979, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/5c5435d4f66f4df785fb17109cd9aa18] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=36.2 K 2024-12-02T14:21:15,064 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/0244e0e233524b9a90ecc236baaca543, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c38e5f528f564213b081ca9bc77c03b8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8862fd315a5c45bbb89ef4f90d36998c] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=38.6 K 2024-12-02T14:21:15,064 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 0711e73ab342430aa723494805e245da, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733149270233 2024-12-02T14:21:15,064 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0244e0e233524b9a90ecc236baaca543, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733149270233 2024-12-02T14:21:15,064 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 81f6363da23941c4b2795dc6551df979, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733149270922 2024-12-02T14:21:15,064 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting c38e5f528f564213b081ca9bc77c03b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733149270922 2024-12-02T14:21:15,064 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c5435d4f66f4df785fb17109cd9aa18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733149273220 2024-12-02T14:21:15,064 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8862fd315a5c45bbb89ef4f90d36998c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733149273205 2024-12-02T14:21:15,084 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#B#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:15,084 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/69cc0a4e39a442159d97abdd9571aa75 is 50, key is test_row_0/B:col10/1733149273814/Put/seqid=0 2024-12-02T14:21:15,085 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#A#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:15,085 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/c2c7c7e42dda415fbe5c90ad67aacc07 is 50, key is test_row_0/A:col10/1733149273814/Put/seqid=0 2024-12-02T14:21:15,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T14:21:15,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:15,086 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T14:21:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:15,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/143895dd57744e09b02d068a07539a77 is 50, key is test_row_0/A:col10/1733149273827/Put/seqid=0 2024-12-02T14:21:15,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742300_1476 (size=12847) 2024-12-02T14:21:15,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742299_1475 (size=12847) 2024-12-02T14:21:15,106 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/c2c7c7e42dda415fbe5c90ad67aacc07 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c2c7c7e42dda415fbe5c90ad67aacc07 2024-12-02T14:21:15,106 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/69cc0a4e39a442159d97abdd9571aa75 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/69cc0a4e39a442159d97abdd9571aa75 2024-12-02T14:21:15,114 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/A of 9c93976ab5d5804c690c7e75c9981484 into c2c7c7e42dda415fbe5c90ad67aacc07(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:15,114 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:15,114 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/A, priority=13, startTime=1733149275063; duration=0sec 2024-12-02T14:21:15,114 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:15,114 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:A 2024-12-02T14:21:15,114 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:15,116 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/B of 9c93976ab5d5804c690c7e75c9981484 into 69cc0a4e39a442159d97abdd9571aa75(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:15,116 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:15,116 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/B, priority=13, startTime=1733149275063; duration=0sec 2024-12-02T14:21:15,116 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:15,116 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:B 2024-12-02T14:21:15,117 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:15,117 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/C is initiating minor compaction (all files) 2024-12-02T14:21:15,117 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/C in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:15,117 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/89cd22ab716641538ca1790d6d839f4a, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/5a37891eac72493a9538cb59460aaef9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/4ccb00362b804ea7b7f0f105efd68873] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=36.2 K 2024-12-02T14:21:15,118 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89cd22ab716641538ca1790d6d839f4a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733149270233 2024-12-02T14:21:15,118 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a37891eac72493a9538cb59460aaef9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733149270922 2024-12-02T14:21:15,118 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ccb00362b804ea7b7f0f105efd68873, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733149273220 2024-12-02T14:21:15,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742301_1477 (size=12301) 2024-12-02T14:21:15,124 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/143895dd57744e09b02d068a07539a77 2024-12-02T14:21:15,132 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#C#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:15,132 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/4e78a3aee7894b9c956e34adb442c71e is 50, key is test_row_0/C:col10/1733149273814/Put/seqid=0 2024-12-02T14:21:15,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/88c462d5fe014246bbaaccc0a4ca9d56 is 50, key is test_row_0/B:col10/1733149273827/Put/seqid=0 2024-12-02T14:21:15,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742302_1478 (size=12847) 2024-12-02T14:21:15,155 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/4e78a3aee7894b9c956e34adb442c71e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/4e78a3aee7894b9c956e34adb442c71e 2024-12-02T14:21:15,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742303_1479 (size=12301) 2024-12-02T14:21:15,161 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/88c462d5fe014246bbaaccc0a4ca9d56 2024-12-02T14:21:15,162 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/C of 9c93976ab5d5804c690c7e75c9981484 into 4e78a3aee7894b9c956e34adb442c71e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:15,162 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:15,162 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/C, priority=13, startTime=1733149275063; duration=0sec 2024-12-02T14:21:15,162 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:15,162 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:C 2024-12-02T14:21:15,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/dc3ecb049f6649d8be97b8f568e054e3 is 50, key is test_row_0/C:col10/1733149273827/Put/seqid=0 2024-12-02T14:21:15,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742304_1480 (size=12301) 2024-12-02T14:21:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:15,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:15,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733149275409, max timestamp=1733149275430Num uniques 859; 2024-12-02T14:21:15,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733149275409, max timestamp=1733149275430Num uniques 859; 2024-12-02T14:21:15,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733149275409, max timestamp=1733149275430Num uniques 859; 2024-12-02T14:21:15,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T14:21:15,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T14:21:15,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T14:21:15,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149335485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:15,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:15,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149335487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:15,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:15,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149335489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:15,587 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/dc3ecb049f6649d8be97b8f568e054e3 2024-12-02T14:21:15,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/143895dd57744e09b02d068a07539a77 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/143895dd57744e09b02d068a07539a77 2024-12-02T14:21:15,597 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/143895dd57744e09b02d068a07539a77, entries=150, sequenceid=283, filesize=12.0 K 2024-12-02T14:21:15,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/88c462d5fe014246bbaaccc0a4ca9d56 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/88c462d5fe014246bbaaccc0a4ca9d56 2024-12-02T14:21:15,602 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/88c462d5fe014246bbaaccc0a4ca9d56, entries=150, sequenceid=283, filesize=12.0 K 2024-12-02T14:21:15,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/dc3ecb049f6649d8be97b8f568e054e3 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/dc3ecb049f6649d8be97b8f568e054e3 2024-12-02T14:21:15,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:15,605 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/dc3ecb049f6649d8be97b8f568e054e3, entries=150, sequenceid=283, filesize=12.0 K 2024-12-02T14:21:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149335598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:15,607 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=161.02 KB/164880 for 9c93976ab5d5804c690c7e75c9981484 in 521ms, sequenceid=283, compaction requested=false 2024-12-02T14:21:15,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:15,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:15,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-02T14:21:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-02T14:21:15,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-02T14:21:15,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8980 sec 2024-12-02T14:21:15,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.9030 sec 2024-12-02T14:21:15,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=167.72 KB heapSize=466.55 KB 2024-12-02T14:21:15,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:15,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T14:21:15,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:15,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T14:21:15,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:15,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T14:21:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:15,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/8e01c3e170e84d1fb9e8a4e05d0108cc is 50, key is test_row_0/A:col10/1733149275618/Put/seqid=0 2024-12-02T14:21:15,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742305_1481 (size=14741) 2024-12-02T14:21:15,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149335630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:15,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149335634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:15,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149335738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:15,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:15,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149335749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:15,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:15,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149335808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T14:21:15,859 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:15,859 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:21:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-02T14:21:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T14:21:15,861 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:21:15,861 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:21:15,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:21:15,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149335949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:15,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149335959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T14:21:16,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T14:21:16,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:16,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:16,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:16,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:16,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:16,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149336008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:16,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149336009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:16,015 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 9c93976ab5d5804c690c7e75c9981484 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:16,015 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 9c93976ab5d5804c690c7e75c9981484 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:16,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/8e01c3e170e84d1fb9e8a4e05d0108cc 2024-12-02T14:21:16,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/59a947dbbd69491eb1f66f852392ed5f is 50, key is test_row_0/B:col10/1733149275618/Put/seqid=0 2024-12-02T14:21:16,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742306_1482 (size=12301) 2024-12-02T14:21:16,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/59a947dbbd69491eb1f66f852392ed5f 2024-12-02T14:21:16,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/ee8880e272fb4459b80af3e52e6db37d is 50, key is test_row_0/C:col10/1733149275618/Put/seqid=0 2024-12-02T14:21:16,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742307_1483 (size=12301) 2024-12-02T14:21:16,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149336119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:16,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T14:21:16,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:16,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:16,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:16,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:16,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T14:21:16,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149336258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:16,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149336268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:16,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T14:21:16,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:16,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:16,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:16,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:16,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:16,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/ee8880e272fb4459b80af3e52e6db37d 2024-12-02T14:21:16,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/8e01c3e170e84d1fb9e8a4e05d0108cc as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8e01c3e170e84d1fb9e8a4e05d0108cc 2024-12-02T14:21:16,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8e01c3e170e84d1fb9e8a4e05d0108cc, entries=200, sequenceid=324, filesize=14.4 K 2024-12-02T14:21:16,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/59a947dbbd69491eb1f66f852392ed5f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/59a947dbbd69491eb1f66f852392ed5f 2024-12-02T14:21:16,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/59a947dbbd69491eb1f66f852392ed5f, entries=150, sequenceid=324, filesize=12.0 K 2024-12-02T14:21:16,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/ee8880e272fb4459b80af3e52e6db37d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/ee8880e272fb4459b80af3e52e6db37d 2024-12-02T14:21:16,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/ee8880e272fb4459b80af3e52e6db37d, entries=150, sequenceid=324, filesize=12.0 K 2024-12-02T14:21:16,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~466.50 KB/477696, currentSize=20.13 KB/20610 for 9c93976ab5d5804c690c7e75c9981484 in 841ms, sequenceid=324, compaction requested=true 2024-12-02T14:21:16,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:16,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:21:16,460 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:16,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:16,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:21:16,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:16,460 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:16,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:21:16,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:16,461 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:16,461 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/B is initiating minor compaction (all files) 2024-12-02T14:21:16,461 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/B in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:16,461 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/69cc0a4e39a442159d97abdd9571aa75, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/88c462d5fe014246bbaaccc0a4ca9d56, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/59a947dbbd69491eb1f66f852392ed5f] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=36.6 K 2024-12-02T14:21:16,461 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 69cc0a4e39a442159d97abdd9571aa75, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733149273220 2024-12-02T14:21:16,462 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 88c462d5fe014246bbaaccc0a4ca9d56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733149273826 2024-12-02T14:21:16,463 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 59a947dbbd69491eb1f66f852392ed5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733149275483 2024-12-02T14:21:16,463 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:16,463 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/A is initiating minor compaction (all files) 2024-12-02T14:21:16,463 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/A in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:16,463 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c2c7c7e42dda415fbe5c90ad67aacc07, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/143895dd57744e09b02d068a07539a77, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8e01c3e170e84d1fb9e8a4e05d0108cc] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=39.0 K 2024-12-02T14:21:16,463 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2c7c7e42dda415fbe5c90ad67aacc07, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733149273220 2024-12-02T14:21:16,464 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 143895dd57744e09b02d068a07539a77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733149273826 2024-12-02T14:21:16,464 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e01c3e170e84d1fb9e8a4e05d0108cc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733149275483 2024-12-02T14:21:16,470 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:16,471 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/540d7d1bce184ec5a9ffab385154cdf6 is 50, key is test_row_0/B:col10/1733149275618/Put/seqid=0 2024-12-02T14:21:16,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T14:21:16,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:16,472 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-02T14:21:16,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:16,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:16,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:16,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:16,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:16,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:16,475 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#A#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:16,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742308_1484 (size=12949) 2024-12-02T14:21:16,475 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/7f34dd9e339249238c5f0e3d9ea078cd is 50, key is test_row_0/A:col10/1733149275618/Put/seqid=0 2024-12-02T14:21:16,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/873ad2aa7aec4cfa8c2277576469f113 is 50, key is test_row_0/A:col10/1733149275633/Put/seqid=0 2024-12-02T14:21:16,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742309_1485 (size=12949) 2024-12-02T14:21:16,482 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/540d7d1bce184ec5a9ffab385154cdf6 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/540d7d1bce184ec5a9ffab385154cdf6 2024-12-02T14:21:16,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742310_1486 (size=9857) 2024-12-02T14:21:16,486 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/B of 9c93976ab5d5804c690c7e75c9981484 into 540d7d1bce184ec5a9ffab385154cdf6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:16,486 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:16,486 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/B, priority=13, startTime=1733149276460; duration=0sec 2024-12-02T14:21:16,486 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:16,486 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:B 2024-12-02T14:21:16,486 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:16,487 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:16,487 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/C is initiating minor compaction (all files) 2024-12-02T14:21:16,487 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/C in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:16,487 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/4e78a3aee7894b9c956e34adb442c71e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/dc3ecb049f6649d8be97b8f568e054e3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/ee8880e272fb4459b80af3e52e6db37d] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=36.6 K 2024-12-02T14:21:16,487 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e78a3aee7894b9c956e34adb442c71e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733149273220 2024-12-02T14:21:16,487 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/7f34dd9e339249238c5f0e3d9ea078cd as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/7f34dd9e339249238c5f0e3d9ea078cd 2024-12-02T14:21:16,487 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting dc3ecb049f6649d8be97b8f568e054e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733149273826 2024-12-02T14:21:16,488 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting ee8880e272fb4459b80af3e52e6db37d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733149275483 2024-12-02T14:21:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T14:21:16,491 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/A of 9c93976ab5d5804c690c7e75c9981484 into 7f34dd9e339249238c5f0e3d9ea078cd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:16,491 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:16,491 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/A, priority=13, startTime=1733149276460; duration=0sec 2024-12-02T14:21:16,491 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:16,491 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:A 2024-12-02T14:21:16,493 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#C#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:16,493 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/5dd9d850ca814c689b9ae831542867bc is 50, key is test_row_0/C:col10/1733149275618/Put/seqid=0 2024-12-02T14:21:16,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742311_1487 (size=12949) 2024-12-02T14:21:16,505 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/5dd9d850ca814c689b9ae831542867bc as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/5dd9d850ca814c689b9ae831542867bc 2024-12-02T14:21:16,510 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/C of 9c93976ab5d5804c690c7e75c9981484 into 5dd9d850ca814c689b9ae831542867bc(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:16,510 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:16,510 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/C, priority=13, startTime=1733149276460; duration=0sec 2024-12-02T14:21:16,510 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:16,510 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:C 2024-12-02T14:21:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:16,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:16,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149276673, max timestamp=1733149276708Num uniques 859; 2024-12-02T14:21:16,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:21:16,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149276673, max timestamp=1733149276708Num uniques 859; 2024-12-02T14:21:16,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:21:16,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733149276673, max timestamp=1733149276708Num uniques 859; 2024-12-02T14:21:16,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T14:21:16,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733149276773, max timestamp=1733149276800Num uniques 1018; 2024-12-02T14:21:16,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-02T14:21:16,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733149276773, max timestamp=1733149276800Num uniques 1018; 2024-12-02T14:21:16,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733149276773, max timestamp=1733149276800Num uniques 1018; 2024-12-02T14:21:16,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-02T14:21:16,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-02T14:21:16,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733149276826, max timestamp=1733149276836Num uniques 1177; 2024-12-02T14:21:16,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-12-02T14:21:16,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733149276826, max timestamp=1733149276836Num uniques 1177; 2024-12-02T14:21:16,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-12-02T14:21:16,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733149276826, max timestamp=1733149276836Num uniques 1177; 2024-12-02T14:21:16,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-12-02T14:21:16,883 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/873ad2aa7aec4cfa8c2277576469f113 2024-12-02T14:21:16,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733149276873, max timestamp=1733149276882Num uniques 1136; 2024-12-02T14:21:16,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-02T14:21:16,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733149276873, max timestamp=1733149276882Num uniques 1136; 2024-12-02T14:21:16,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-02T14:21:16,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733149276873, max timestamp=1733149276882Num uniques 1136; 2024-12-02T14:21:16,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-02T14:21:16,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/acdaa0d320814c63802436f0f679b433 is 50, key is test_row_0/B:col10/1733149275633/Put/seqid=0 2024-12-02T14:21:16,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742312_1488 (size=9857) 2024-12-02T14:21:16,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733149276920, max timestamp=1733149276935Num uniques 1045; 2024-12-02T14:21:16,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733149276920, max timestamp=1733149276935Num uniques 1045; 2024-12-02T14:21:16,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-02T14:21:16,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-02T14:21:16,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733149276920, max timestamp=1733149276935Num uniques 1045; 2024-12-02T14:21:16,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-02T14:21:16,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:16,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733149276965, max timestamp=1733149276984Num uniques 1154; 2024-12-02T14:21:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149336976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:16,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733149276965, max timestamp=1733149276984Num uniques 1154; 2024-12-02T14:21:16,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=13920, suffix heap size=147984,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-12-02T14:21:16,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=13920, suffix heap size=147984,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-12-02T14:21:16,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:16,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:16,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:16,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733149276965, max timestamp=1733149276984Num uniques 1154; 2024-12-02T14:21:16,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=13920, suffix heap size=147984,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-12-02T14:21:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T14:21:17,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149337075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:17,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:17,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733149277028, max timestamp=1733149277076Num uniques 1163; 2024-12-02T14:21:17,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=7458, suffix heap size=148920,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=163 2024-12-02T14:21:17,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733149277028, max timestamp=1733149277076Num uniques 1163; 2024-12-02T14:21:17,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=7458, suffix heap size=148920,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=163 2024-12-02T14:21:17,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733149277028, max timestamp=1733149277076Num uniques 1163; 2024-12-02T14:21:17,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=7458, suffix heap size=148920,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=163 2024-12-02T14:21:17,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733149277121, max timestamp=1733149277143Num uniques 1022; 2024-12-02T14:21:17,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-02T14:21:17,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733149277121, max timestamp=1733149277143Num uniques 1022; 2024-12-02T14:21:17,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-02T14:21:17,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733149277121, max timestamp=1733149277143Num uniques 1022; 2024-12-02T14:21:17,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-02T14:21:17,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1733149277199, max timestamp=1733149277224Num uniques 1031; 2024-12-02T14:21:17,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1733149277199, max timestamp=1733149277224Num uniques 1031; 2024-12-02T14:21:17,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-12-02T14:21:17,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-12-02T14:21:17,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1733149277199, max timestamp=1733149277224Num uniques 1031; 2024-12-02T14:21:17,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-12-02T14:21:17,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733149277255, max timestamp=1733149277270Num uniques 1140; 2024-12-02T14:21:17,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733149277255, max timestamp=1733149277270Num uniques 1140; 2024-12-02T14:21:17,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733149277255, max timestamp=1733149277270Num uniques 1140; 2024-12-02T14:21:17,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-12-02T14:21:17,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-12-02T14:21:17,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-12-02T14:21:17,304 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/acdaa0d320814c63802436f0f679b433 2024-12-02T14:21:17,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/22fe181c85134d30a0ad0d7d0958834d is 50, key is test_row_0/C:col10/1733149275633/Put/seqid=0 2024-12-02T14:21:17,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742313_1489 (size=9857) 2024-12-02T14:21:17,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733149277296, max timestamp=1733149277316Num uniques 1099; 2024-12-02T14:21:17,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-12-02T14:21:17,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733149277296, max timestamp=1733149277316Num uniques 1099; 2024-12-02T14:21:17,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-12-02T14:21:17,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733149277296, max timestamp=1733149277316Num uniques 1099; 2024-12-02T14:21:17,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-12-02T14:21:17,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733149277359, max timestamp=1733149277377Num uniques 1108; 2024-12-02T14:21:17,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-12-02T14:21:17,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733149277359, max timestamp=1733149277377Num uniques 1108; 2024-12-02T14:21:17,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-12-02T14:21:17,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733149277359, max timestamp=1733149277377Num uniques 1108; 2024-12-02T14:21:17,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-12-02T14:21:17,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733149277408, max timestamp=1733149277424Num uniques 1017; 2024-12-02T14:21:17,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733149277408, max timestamp=1733149277424Num uniques 1017; 2024-12-02T14:21:17,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733149277408, max timestamp=1733149277424Num uniques 1017; 2024-12-02T14:21:17,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-12-02T14:21:17,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-12-02T14:21:17,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=7642, suffix heap size=133736,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=167 2024-12-02T14:21:17,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733149277464, max timestamp=1733149277475Num uniques 1026; 2024-12-02T14:21:17,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-12-02T14:21:17,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733149277464, max timestamp=1733149277475Num uniques 1026; 2024-12-02T14:21:17,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733149277464, max timestamp=1733149277475Num uniques 1026; 2024-12-02T14:21:17,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-12-02T14:21:17,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-12-02T14:21:17,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733149277499, max timestamp=1733149277521Num uniques 1035; 2024-12-02T14:21:17,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733149277499, max timestamp=1733149277521Num uniques 1035; 2024-12-02T14:21:17,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=13050, suffix heap size=135608,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=285 2024-12-02T14:21:17,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=13050, suffix heap size=135608,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=285 2024-12-02T14:21:17,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733149277499, max timestamp=1733149277521Num uniques 1035; 2024-12-02T14:21:17,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=13050, suffix heap size=135608,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=285 2024-12-02T14:21:17,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1733149277581, max timestamp=1733149277599Num uniques 1144; 2024-12-02T14:21:17,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=13464, suffix heap size=146944,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=294 2024-12-02T14:21:17,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1733149277581, max timestamp=1733149277599Num uniques 1144; 2024-12-02T14:21:17,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:17,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=13464, suffix heap size=146944,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=294 2024-12-02T14:21:17,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149337593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:17,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1733149277581, max timestamp=1733149277599Num uniques 1144; 2024-12-02T14:21:17,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=13464, suffix heap size=146944,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=294 2024-12-02T14:21:17,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:17,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:17,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:17,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733149277628, max timestamp=1733149277656Num uniques 1153; 2024-12-02T14:21:17,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733149277628, max timestamp=1733149277656Num uniques 1153; 2024-12-02T14:21:17,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=9295, suffix heap size=147880,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=203 2024-12-02T14:21:17,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=9295, suffix heap size=147880,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=203 2024-12-02T14:21:17,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1733149277628, max timestamp=1733149277656Num uniques 1153; 2024-12-02T14:21:17,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52805, new segment data size=9295, suffix heap size=147880,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1153, new segment cells count=203 2024-12-02T14:21:17,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1733149277685, max timestamp=1733149277703Num uniques 1062; 2024-12-02T14:21:17,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=9702, suffix heap size=138416,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=212 2024-12-02T14:21:17,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1733149277685, max timestamp=1733149277703Num uniques 1062; 2024-12-02T14:21:17,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=9702, suffix heap size=138416,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=212 2024-12-02T14:21:17,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1733149277685, max timestamp=1733149277703Num uniques 1062; 2024-12-02T14:21:17,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=9702, suffix heap size=138416,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=212 2024-12-02T14:21:17,714 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/22fe181c85134d30a0ad0d7d0958834d 2024-12-02T14:21:17,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/873ad2aa7aec4cfa8c2277576469f113 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/873ad2aa7aec4cfa8c2277576469f113 2024-12-02T14:21:17,721 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/873ad2aa7aec4cfa8c2277576469f113, entries=100, sequenceid=330, filesize=9.6 K 2024-12-02T14:21:17,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/acdaa0d320814c63802436f0f679b433 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/acdaa0d320814c63802436f0f679b433 2024-12-02T14:21:17,724 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/acdaa0d320814c63802436f0f679b433, entries=100, sequenceid=330, filesize=9.6 K 2024-12-02T14:21:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/22fe181c85134d30a0ad0d7d0958834d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/22fe181c85134d30a0ad0d7d0958834d 2024-12-02T14:21:17,728 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/22fe181c85134d30a0ad0d7d0958834d, entries=100, sequenceid=330, filesize=9.6 K 2024-12-02T14:21:17,729 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=93.93 KB/96180 for 9c93976ab5d5804c690c7e75c9981484 in 1257ms, sequenceid=330, compaction requested=false 2024-12-02T14:21:17,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:17,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:17,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-02T14:21:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-02T14:21:17,731 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-02T14:21:17,731 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8690 sec 2024-12-02T14:21:17,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.8730 sec 2024-12-02T14:21:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:17,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=100.63 KB heapSize=280.41 KB 2024-12-02T14:21:17,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:17,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:21:17,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:17,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:21:17,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:17,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:21:17,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/477a07a97fcc4ef3b7aba64204580399 is 50, key is test_row_0/A:col10/1733149277726/Put/seqid=0 2024-12-02T14:21:17,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742314_1490 (size=12301) 2024-12-02T14:21:17,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149337755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:17,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149337763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:17,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:17,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149337764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:17,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149337869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:17,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149337869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:17,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149337878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T14:21:18,008 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:18,009 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:21:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-02T14:21:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T14:21:18,010 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:21:18,011 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:21:18,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:21:18,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149338028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:18,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149338028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:18,037 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 9c93976ab5d5804c690c7e75c9981484 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:18,037 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 9c93976ab5d5804c690c7e75c9981484 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:18,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149338078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:18,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149338078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:18,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:18,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:18,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149338088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:18,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T14:21:18,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/477a07a97fcc4ef3b7aba64204580399 2024-12-02T14:21:18,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/4734a5f4277a4ec584602c06302a6e28 is 50, key is test_row_0/B:col10/1733149277726/Put/seqid=0 2024-12-02T14:21:18,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742315_1491 (size=12301) 2024-12-02T14:21:18,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/4734a5f4277a4ec584602c06302a6e28 2024-12-02T14:21:18,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-02T14:21:18,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:18,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:18,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:18,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:18,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:18,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/7198d7e88da14187becc08da2600632f is 50, key is test_row_0/C:col10/1733149277726/Put/seqid=0 2024-12-02T14:21:18,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742316_1492 (size=12301) 2024-12-02T14:21:18,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=657 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/7198d7e88da14187becc08da2600632f 2024-12-02T14:21:18,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/477a07a97fcc4ef3b7aba64204580399 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/477a07a97fcc4ef3b7aba64204580399 2024-12-02T14:21:18,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/477a07a97fcc4ef3b7aba64204580399, entries=150, sequenceid=657, filesize=12.0 K 2024-12-02T14:21:18,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/4734a5f4277a4ec584602c06302a6e28 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/4734a5f4277a4ec584602c06302a6e28 2024-12-02T14:21:18,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/4734a5f4277a4ec584602c06302a6e28, entries=150, sequenceid=657, filesize=12.0 K 2024-12-02T14:21:18,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/7198d7e88da14187becc08da2600632f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/7198d7e88da14187becc08da2600632f 2024-12-02T14:21:18,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/7198d7e88da14187becc08da2600632f, entries=150, sequenceid=657, filesize=12.0 K 2024-12-02T14:21:18,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~300.28 KB/307488, currentSize=73.80 KB/75570 for 9c93976ab5d5804c690c7e75c9981484 in 459ms, sequenceid=657, compaction requested=true 2024-12-02T14:21:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:21:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:18,193 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:21:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:21:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T14:21:18,194 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:18,194 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:18,194 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/A is initiating minor compaction (all files) 2024-12-02T14:21:18,194 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:18,194 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/A in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:18,194 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/B is initiating minor compaction (all files) 2024-12-02T14:21:18,194 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/B in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:18,194 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/7f34dd9e339249238c5f0e3d9ea078cd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/873ad2aa7aec4cfa8c2277576469f113, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/477a07a97fcc4ef3b7aba64204580399] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=34.3 K 2024-12-02T14:21:18,195 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/540d7d1bce184ec5a9ffab385154cdf6, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/acdaa0d320814c63802436f0f679b433, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/4734a5f4277a4ec584602c06302a6e28] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=34.3 K 2024-12-02T14:21:18,197 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f34dd9e339249238c5f0e3d9ea078cd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733149275483 2024-12-02T14:21:18,197 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 540d7d1bce184ec5a9ffab385154cdf6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733149275483 2024-12-02T14:21:18,197 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 873ad2aa7aec4cfa8c2277576469f113, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733149275628 2024-12-02T14:21:18,197 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting acdaa0d320814c63802436f0f679b433, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733149275628 2024-12-02T14:21:18,197 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 477a07a97fcc4ef3b7aba64204580399, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1733149277726 2024-12-02T14:21:18,197 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4734a5f4277a4ec584602c06302a6e28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1733149277726 2024-12-02T14:21:18,203 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#B#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:18,204 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/f173177a5ddc45a7bad29d1d934689f9 is 50, key is test_row_0/B:col10/1733149277726/Put/seqid=0 2024-12-02T14:21:18,206 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#A#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:18,207 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/b7e2c58acede44db90973ca5403f8b41 is 50, key is test_row_0/A:col10/1733149277726/Put/seqid=0 2024-12-02T14:21:18,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742318_1494 (size=13051) 2024-12-02T14:21:18,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742317_1493 (size=13051) 2024-12-02T14:21:18,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-02T14:21:18,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:18,316 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T14:21:18,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:18,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:18,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:18,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:18,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:18,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:18,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/312221c0f6a14adb9f5d7eb6d3aaed94 is 50, key is test_row_0/A:col10/1733149277762/Put/seqid=0 2024-12-02T14:21:18,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742319_1495 (size=12301) 2024-12-02T14:21:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T14:21:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:18,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:18,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149338422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:18,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149338422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:18,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149338423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:18,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149338538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149338538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:18,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149338538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:18,619 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/b7e2c58acede44db90973ca5403f8b41 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/b7e2c58acede44db90973ca5403f8b41 2024-12-02T14:21:18,619 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/f173177a5ddc45a7bad29d1d934689f9 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/f173177a5ddc45a7bad29d1d934689f9 2024-12-02T14:21:18,623 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/B of 9c93976ab5d5804c690c7e75c9981484 into f173177a5ddc45a7bad29d1d934689f9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:18,623 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/A of 9c93976ab5d5804c690c7e75c9981484 into b7e2c58acede44db90973ca5403f8b41(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:18,623 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:18,623 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:18,623 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/B, priority=13, startTime=1733149278193; duration=0sec 2024-12-02T14:21:18,623 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/A, priority=13, startTime=1733149278193; duration=0sec 2024-12-02T14:21:18,623 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:18,623 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:18,623 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:A 2024-12-02T14:21:18,623 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:B 2024-12-02T14:21:18,623 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:18,624 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:18,624 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/C is initiating minor compaction (all files) 2024-12-02T14:21:18,624 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/C in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:18,624 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/5dd9d850ca814c689b9ae831542867bc, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/22fe181c85134d30a0ad0d7d0958834d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/7198d7e88da14187becc08da2600632f] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=34.3 K 2024-12-02T14:21:18,625 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dd9d850ca814c689b9ae831542867bc, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733149275483 2024-12-02T14:21:18,625 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22fe181c85134d30a0ad0d7d0958834d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733149275628 2024-12-02T14:21:18,625 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7198d7e88da14187becc08da2600632f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1733149277726 2024-12-02T14:21:18,632 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#C#compaction#420 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:18,632 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/c2ae3d56569c4f688aaf4a44ec85f2c9 is 50, key is test_row_0/C:col10/1733149277726/Put/seqid=0 2024-12-02T14:21:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T14:21:18,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742320_1496 (size=13051) 2024-12-02T14:21:18,727 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/312221c0f6a14adb9f5d7eb6d3aaed94 2024-12-02T14:21:18,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/a62b8a12769242119eb320668f3576e8 is 50, key is test_row_0/B:col10/1733149277762/Put/seqid=0 2024-12-02T14:21:18,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742321_1497 (size=12301) 2024-12-02T14:21:18,735 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/a62b8a12769242119eb320668f3576e8 2024-12-02T14:21:18,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/aab0bcffb50d4393becadc67e3b874d2 is 50, key is test_row_0/C:col10/1733149277762/Put/seqid=0 2024-12-02T14:21:18,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742322_1498 (size=12301) 2024-12-02T14:21:18,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149338749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149338749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:18,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149338749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:19,062 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/c2ae3d56569c4f688aaf4a44ec85f2c9 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/c2ae3d56569c4f688aaf4a44ec85f2c9 2024-12-02T14:21:19,066 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/C of 9c93976ab5d5804c690c7e75c9981484 into c2ae3d56569c4f688aaf4a44ec85f2c9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:19,066 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:19,066 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/C, priority=13, startTime=1733149278193; duration=0sec 2024-12-02T14:21:19,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:19,066 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:19,066 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:C 2024-12-02T14:21:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149339058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:19,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149339058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149339058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:19,145 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=671 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/aab0bcffb50d4393becadc67e3b874d2 2024-12-02T14:21:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T14:21:19,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/312221c0f6a14adb9f5d7eb6d3aaed94 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/312221c0f6a14adb9f5d7eb6d3aaed94 2024-12-02T14:21:19,151 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/312221c0f6a14adb9f5d7eb6d3aaed94, entries=150, sequenceid=671, filesize=12.0 K 2024-12-02T14:21:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/a62b8a12769242119eb320668f3576e8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/a62b8a12769242119eb320668f3576e8 2024-12-02T14:21:19,154 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/a62b8a12769242119eb320668f3576e8, entries=150, sequenceid=671, filesize=12.0 K 2024-12-02T14:21:19,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/aab0bcffb50d4393becadc67e3b874d2 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/aab0bcffb50d4393becadc67e3b874d2 2024-12-02T14:21:19,157 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/aab0bcffb50d4393becadc67e3b874d2, entries=150, sequenceid=671, filesize=12.0 K 2024-12-02T14:21:19,157 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9c93976ab5d5804c690c7e75c9981484 in 841ms, sequenceid=671, compaction requested=false 2024-12-02T14:21:19,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:19,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:19,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-02T14:21:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-02T14:21:19,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-02T14:21:19,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1470 sec 2024-12-02T14:21:19,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.1510 sec 2024-12-02T14:21:19,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:19,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T14:21:19,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:19,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:19,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:19,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:19,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:19,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:19,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/70cb2d2e42b94bdfb6aad51d48742421 is 50, key is test_row_0/A:col10/1733149279579/Put/seqid=0 2024-12-02T14:21:19,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742323_1499 (size=14741) 2024-12-02T14:21:19,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149339600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:19,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:19,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:19,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149339606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149339606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:19,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149339718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:19,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149339718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:19,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149339718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:19,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149339929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:19,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149339929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:19,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149339929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:19,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=693 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/70cb2d2e42b94bdfb6aad51d48742421 2024-12-02T14:21:19,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/626e4ffb2a8c4fa6aff601c97a600000 is 50, key is test_row_0/B:col10/1733149279579/Put/seqid=0 2024-12-02T14:21:20,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742324_1500 (size=12301) 2024-12-02T14:21:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T14:21:20,159 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:20,159 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:21:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-02T14:21:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T14:21:20,161 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:21:20,162 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:21:20,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:21:20,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149340238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:20,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:20,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:20,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:20,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149340249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:20,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149340249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T14:21:20,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T14:21:20,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:20,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:20,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:20,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:20,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:20,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=693 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/626e4ffb2a8c4fa6aff601c97a600000 2024-12-02T14:21:20,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/f79b50d636e841af8b7d0f911977809e is 50, key is test_row_0/C:col10/1733149279579/Put/seqid=0 2024-12-02T14:21:20,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742325_1501 (size=12301) 2024-12-02T14:21:20,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T14:21:20,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:20,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:20,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:20,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:20,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T14:21:20,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T14:21:20,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:20,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:20,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:20,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:20,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:20,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149340759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:20,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:20,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:20,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:20,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149340768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:20,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:20,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:20,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:20,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T14:21:20,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:20,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. as already flushing 2024-12-02T14:21:20,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:20,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:20,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:20,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:20,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149340778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T14:21:20,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=693 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/f79b50d636e841af8b7d0f911977809e 2024-12-02T14:21:20,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/70cb2d2e42b94bdfb6aad51d48742421 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/70cb2d2e42b94bdfb6aad51d48742421 2024-12-02T14:21:20,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/70cb2d2e42b94bdfb6aad51d48742421, entries=200, sequenceid=693, filesize=14.4 K 2024-12-02T14:21:20,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/626e4ffb2a8c4fa6aff601c97a600000 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/626e4ffb2a8c4fa6aff601c97a600000 2024-12-02T14:21:20,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/626e4ffb2a8c4fa6aff601c97a600000, entries=150, sequenceid=693, filesize=12.0 K 2024-12-02T14:21:20,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/f79b50d636e841af8b7d0f911977809e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/f79b50d636e841af8b7d0f911977809e 2024-12-02T14:21:20,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/f79b50d636e841af8b7d0f911977809e, entries=150, sequenceid=693, filesize=12.0 K 2024-12-02T14:21:20,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 9c93976ab5d5804c690c7e75c9981484 in 1265ms, sequenceid=693, compaction requested=true 2024-12-02T14:21:20,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:20,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:21:20,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:20,845 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:20,845 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:20,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:21:20,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:20,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:21:20,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:20,846 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:20,846 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:20,846 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/B is initiating minor compaction (all files) 2024-12-02T14:21:20,846 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/A is initiating minor compaction (all files) 2024-12-02T14:21:20,846 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/B in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:20,846 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/A in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:20,846 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/b7e2c58acede44db90973ca5403f8b41, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/312221c0f6a14adb9f5d7eb6d3aaed94, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/70cb2d2e42b94bdfb6aad51d48742421] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=39.2 K 2024-12-02T14:21:20,846 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/f173177a5ddc45a7bad29d1d934689f9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/a62b8a12769242119eb320668f3576e8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/626e4ffb2a8c4fa6aff601c97a600000] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=36.8 K 2024-12-02T14:21:20,847 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting f173177a5ddc45a7bad29d1d934689f9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1733149277726 2024-12-02T14:21:20,847 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7e2c58acede44db90973ca5403f8b41, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1733149277726 2024-12-02T14:21:20,847 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 312221c0f6a14adb9f5d7eb6d3aaed94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1733149277754 2024-12-02T14:21:20,847 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting a62b8a12769242119eb320668f3576e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1733149277754 2024-12-02T14:21:20,848 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70cb2d2e42b94bdfb6aad51d48742421, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=693, earliestPutTs=1733149278420 2024-12-02T14:21:20,848 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 626e4ffb2a8c4fa6aff601c97a600000, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=693, earliestPutTs=1733149278420 2024-12-02T14:21:20,859 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#A#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:20,860 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/49d25d273dab4f958246dede1b575016 is 50, key is test_row_0/A:col10/1733149279579/Put/seqid=0 2024-12-02T14:21:20,863 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#B#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:20,863 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/edbc767234734e429c4e7038bacc255e is 50, key is test_row_0/B:col10/1733149279579/Put/seqid=0 2024-12-02T14:21:20,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742327_1503 (size=13153) 2024-12-02T14:21:20,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742326_1502 (size=13153) 2024-12-02T14:21:20,882 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/edbc767234734e429c4e7038bacc255e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/edbc767234734e429c4e7038bacc255e 2024-12-02T14:21:20,890 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/B of 9c93976ab5d5804c690c7e75c9981484 into edbc767234734e429c4e7038bacc255e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:20,890 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:20,890 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/B, priority=13, startTime=1733149280845; duration=0sec 2024-12-02T14:21:20,890 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:20,890 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:B 2024-12-02T14:21:20,891 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:20,892 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:20,892 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/C is initiating minor compaction (all files) 2024-12-02T14:21:20,892 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/C in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:20,892 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/c2ae3d56569c4f688aaf4a44ec85f2c9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/aab0bcffb50d4393becadc67e3b874d2, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/f79b50d636e841af8b7d0f911977809e] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=36.8 K 2024-12-02T14:21:20,892 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting c2ae3d56569c4f688aaf4a44ec85f2c9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=657, earliestPutTs=1733149277726 2024-12-02T14:21:20,892 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting aab0bcffb50d4393becadc67e3b874d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=671, earliestPutTs=1733149277754 2024-12-02T14:21:20,892 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting f79b50d636e841af8b7d0f911977809e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=693, earliestPutTs=1733149278420 2024-12-02T14:21:20,898 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#C#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:20,898 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/f702737a2521432c96b8f290737c9603 is 50, key is test_row_0/C:col10/1733149279579/Put/seqid=0 2024-12-02T14:21:20,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742328_1504 (size=13153) 2024-12-02T14:21:20,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T14:21:20,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:20,925 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T14:21:20,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:20,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:20,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:20,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:20,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:20,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:20,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/801c21498ef049c8a24b10ebea4d0bd8 is 50, key is test_row_0/A:col10/1733149279605/Put/seqid=0 2024-12-02T14:21:20,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742329_1505 (size=12301) 2024-12-02T14:21:20,939 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=708 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/801c21498ef049c8a24b10ebea4d0bd8 2024-12-02T14:21:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/3b67dfcb2d1243f697b8c2be46165f7d is 50, key is test_row_0/B:col10/1733149279605/Put/seqid=0 2024-12-02T14:21:20,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742330_1506 (size=12301) 2024-12-02T14:21:21,281 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/49d25d273dab4f958246dede1b575016 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/49d25d273dab4f958246dede1b575016 2024-12-02T14:21:21,284 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/A of 9c93976ab5d5804c690c7e75c9981484 into 49d25d273dab4f958246dede1b575016(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:21,284 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:21,284 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/A, priority=13, startTime=1733149280845; duration=0sec 2024-12-02T14:21:21,284 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:21,284 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:A 2024-12-02T14:21:21,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T14:21:21,305 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/f702737a2521432c96b8f290737c9603 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/f702737a2521432c96b8f290737c9603 2024-12-02T14:21:21,310 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/C of 9c93976ab5d5804c690c7e75c9981484 into f702737a2521432c96b8f290737c9603(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:21,310 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:21,310 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/C, priority=13, startTime=1733149280846; duration=0sec 2024-12-02T14:21:21,310 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:21,310 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:C 2024-12-02T14:21:21,355 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=708 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/3b67dfcb2d1243f697b8c2be46165f7d 2024-12-02T14:21:21,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/17d4f3644ab94870b0f9d13fe843ac9d is 50, key is test_row_0/C:col10/1733149279605/Put/seqid=0 2024-12-02T14:21:21,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742331_1507 (size=12301) 2024-12-02T14:21:21,364 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=708 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/17d4f3644ab94870b0f9d13fe843ac9d 2024-12-02T14:21:21,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/801c21498ef049c8a24b10ebea4d0bd8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/801c21498ef049c8a24b10ebea4d0bd8 2024-12-02T14:21:21,372 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/801c21498ef049c8a24b10ebea4d0bd8, entries=150, sequenceid=708, filesize=12.0 K 2024-12-02T14:21:21,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/3b67dfcb2d1243f697b8c2be46165f7d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/3b67dfcb2d1243f697b8c2be46165f7d 2024-12-02T14:21:21,384 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/3b67dfcb2d1243f697b8c2be46165f7d, entries=150, sequenceid=708, filesize=12.0 K 2024-12-02T14:21:21,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/17d4f3644ab94870b0f9d13fe843ac9d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/17d4f3644ab94870b0f9d13fe843ac9d 2024-12-02T14:21:21,387 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/17d4f3644ab94870b0f9d13fe843ac9d, entries=150, sequenceid=708, filesize=12.0 K 2024-12-02T14:21:21,388 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 9c93976ab5d5804c690c7e75c9981484 in 463ms, sequenceid=708, compaction requested=false 2024-12-02T14:21:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-02T14:21:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-02T14:21:21,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-02T14:21:21,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2270 sec 2024-12-02T14:21:21,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.2320 sec 2024-12-02T14:21:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:21,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T14:21:21,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:21,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:21,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:21,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:21,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:21,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:21,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/fb1d1d4777b0475982e7f8b8cc725e5a is 50, key is test_row_0/A:col10/1733149281783/Put/seqid=0 2024-12-02T14:21:21,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742332_1508 (size=14741) 2024-12-02T14:21:21,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149281826, max timestamp=1733149281853Num uniques 859; 2024-12-02T14:21:21,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149281826, max timestamp=1733149281853Num uniques 859; 2024-12-02T14:21:21,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:21:21,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:21:21,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149281826, max timestamp=1733149281853Num uniques 859; 2024-12-02T14:21:21,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:21:21,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149341908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:21,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149341914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:21,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149341918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:21,954 INFO [Thread-1932 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1932. 2024-12-02T14:21:21,954 DEBUG [Thread-1932 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:21:21,954 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:21,954 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:21,954 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:21,957 INFO [Thread-1938 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1938. 2024-12-02T14:21:21,957 DEBUG [Thread-1938 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:21:21,957 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:21,958 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:21,958 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:21,958 INFO [Thread-1934 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1934. 2024-12-02T14:21:21,958 DEBUG [Thread-1934 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:21:21,958 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:21,958 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:21,958 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:21,962 INFO [Thread-1936 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1936. 2024-12-02T14:21:21,962 DEBUG [Thread-1936 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:21:21,962 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:21,962 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:21,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:21,965 INFO [Thread-1930 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1930. 2024-12-02T14:21:21,965 DEBUG [Thread-1930 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:21:21,965 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:21,965 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:21,965 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:22,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149342028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:22,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:22,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149342039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149342039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:22,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38744 deadline: 1733149342048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:22,049 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 9c93976ab5d5804c690c7e75c9981484 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:22,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38676 deadline: 1733149342078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:22,079 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 9c93976ab5d5804c690c7e75c9981484 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:22,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/fb1d1d4777b0475982e7f8b8cc725e5a 2024-12-02T14:21:22,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/d93c87388ba7480e95b4101524c4e36f is 50, key is test_row_0/B:col10/1733149281783/Put/seqid=0 2024-12-02T14:21:22,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742333_1509 (size=12301) 2024-12-02T14:21:22,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149342238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:22,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:22,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149342248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149342248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T14:21:22,309 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:22,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38688 deadline: 1733149342549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:22,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:22,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38700 deadline: 1733149342558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38654 deadline: 1733149342558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 , the old value is region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=2 because the exception is null or not the one we care about 2024-12-02T14:21:22,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/d93c87388ba7480e95b4101524c4e36f 2024-12-02T14:21:22,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/9577858966524eb39e13044ddb58e84e is 50, key is test_row_0/C:col10/1733149281783/Put/seqid=0 2024-12-02T14:21:22,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742334_1510 (size=12301) 2024-12-02T14:21:23,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/9577858966524eb39e13044ddb58e84e 2024-12-02T14:21:23,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/fb1d1d4777b0475982e7f8b8cc725e5a as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/fb1d1d4777b0475982e7f8b8cc725e5a 2024-12-02T14:21:23,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/fb1d1d4777b0475982e7f8b8cc725e5a, entries=200, sequenceid=720, filesize=14.4 K 2024-12-02T14:21:23,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/d93c87388ba7480e95b4101524c4e36f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/d93c87388ba7480e95b4101524c4e36f 2024-12-02T14:21:23,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/d93c87388ba7480e95b4101524c4e36f, entries=150, sequenceid=720, filesize=12.0 K 2024-12-02T14:21:23,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/9577858966524eb39e13044ddb58e84e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/9577858966524eb39e13044ddb58e84e 2024-12-02T14:21:23,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/9577858966524eb39e13044ddb58e84e, entries=150, sequenceid=720, filesize=12.0 K 2024-12-02T14:21:23,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 9c93976ab5d5804c690c7e75c9981484 in 1263ms, sequenceid=720, compaction requested=true 2024-12-02T14:21:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:21:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:21:23,053 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9c93976ab5d5804c690c7e75c9981484:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:21:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:23,053 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:23,054 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:23,054 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/A is initiating minor compaction (all files) 2024-12-02T14:21:23,054 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:23,054 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/B is initiating minor compaction (all files) 2024-12-02T14:21:23,054 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/A in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:23,054 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/B in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:23,054 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/49d25d273dab4f958246dede1b575016, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/801c21498ef049c8a24b10ebea4d0bd8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/fb1d1d4777b0475982e7f8b8cc725e5a] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=39.3 K 2024-12-02T14:21:23,055 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/edbc767234734e429c4e7038bacc255e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/3b67dfcb2d1243f697b8c2be46165f7d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/d93c87388ba7480e95b4101524c4e36f] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=36.9 K 2024-12-02T14:21:23,055 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49d25d273dab4f958246dede1b575016, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=693, earliestPutTs=1733149278420 2024-12-02T14:21:23,055 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting edbc767234734e429c4e7038bacc255e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=693, earliestPutTs=1733149278420 2024-12-02T14:21:23,055 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 801c21498ef049c8a24b10ebea4d0bd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=708, earliestPutTs=1733149279598 2024-12-02T14:21:23,055 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b67dfcb2d1243f697b8c2be46165f7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=708, earliestPutTs=1733149279598 2024-12-02T14:21:23,055 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb1d1d4777b0475982e7f8b8cc725e5a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1733149281779 2024-12-02T14:21:23,055 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting d93c87388ba7480e95b4101524c4e36f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1733149281783 2024-12-02T14:21:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:23,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-02T14:21:23,061 INFO [Thread-1927 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1927. 2024-12-02T14:21:23,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:23,061 DEBUG [Thread-1927 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:21:23,061 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:23,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:21:23,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:23,061 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:23,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:23,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:21:23,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:23,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:21:23,063 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#B#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:23,063 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#A#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:23,063 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/28b20568affc41f790b249a9e91d64cc is 50, key is test_row_0/A:col10/1733149281783/Put/seqid=0 2024-12-02T14:21:23,063 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/4c47bd1072dd4ca6b722afb72f50736a is 50, key is test_row_0/B:col10/1733149281783/Put/seqid=0 2024-12-02T14:21:23,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/f9bc407343c349b3bfa6a0df00314dd8 is 50, key is test_row_0/A:col10/1733149281916/Put/seqid=0 2024-12-02T14:21:23,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742335_1511 (size=13255) 2024-12-02T14:21:23,069 INFO [Thread-1919 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1919. 2024-12-02T14:21:23,069 INFO [Thread-1921 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1921. 2024-12-02T14:21:23,069 DEBUG [Thread-1919 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:21:23,069 DEBUG [Thread-1921 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:21:23,069 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:23,069 DEBUG [Thread-1921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:23,069 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:23,069 DEBUG [Thread-1921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:23,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:23,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:23,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742336_1512 (size=13255) 2024-12-02T14:21:23,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742337_1513 (size=12301) 2024-12-02T14:21:23,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=757 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/f9bc407343c349b3bfa6a0df00314dd8 2024-12-02T14:21:23,475 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/28b20568affc41f790b249a9e91d64cc as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/28b20568affc41f790b249a9e91d64cc 2024-12-02T14:21:23,478 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/4c47bd1072dd4ca6b722afb72f50736a as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/4c47bd1072dd4ca6b722afb72f50736a 2024-12-02T14:21:23,480 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/A of 9c93976ab5d5804c690c7e75c9981484 into 28b20568affc41f790b249a9e91d64cc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:23,480 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:23,480 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/A, priority=13, startTime=1733149283053; duration=0sec 2024-12-02T14:21:23,480 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:23,480 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:A 2024-12-02T14:21:23,480 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:23,481 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:23,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/5b0b794c7c9d46ca854ef837e9acdd0b is 50, key is test_row_0/B:col10/1733149281916/Put/seqid=0 2024-12-02T14:21:23,482 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 9c93976ab5d5804c690c7e75c9981484/C is initiating minor compaction (all files) 2024-12-02T14:21:23,482 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9c93976ab5d5804c690c7e75c9981484/C in TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:23,482 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/f702737a2521432c96b8f290737c9603, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/17d4f3644ab94870b0f9d13fe843ac9d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/9577858966524eb39e13044ddb58e84e] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp, totalSize=36.9 K 2024-12-02T14:21:23,482 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting f702737a2521432c96b8f290737c9603, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=693, earliestPutTs=1733149278420 2024-12-02T14:21:23,482 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17d4f3644ab94870b0f9d13fe843ac9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=708, earliestPutTs=1733149279598 2024-12-02T14:21:23,482 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/B of 9c93976ab5d5804c690c7e75c9981484 into 4c47bd1072dd4ca6b722afb72f50736a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:23,482 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9577858966524eb39e13044ddb58e84e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1733149281783 2024-12-02T14:21:23,482 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:23,483 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/B, priority=13, startTime=1733149283053; duration=0sec 2024-12-02T14:21:23,483 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:23,483 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:B 2024-12-02T14:21:23,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742338_1514 (size=12301) 2024-12-02T14:21:23,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=757 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/5b0b794c7c9d46ca854ef837e9acdd0b 2024-12-02T14:21:23,491 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9c93976ab5d5804c690c7e75c9981484#C#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:23,492 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/56a8ccf048e649d1bd06a9cdd35d7a32 is 50, key is test_row_0/C:col10/1733149281783/Put/seqid=0 2024-12-02T14:21:23,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/8b705214ea5849e0843ae82c289c9dde is 50, key is test_row_0/C:col10/1733149281916/Put/seqid=0 2024-12-02T14:21:23,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742339_1515 (size=13255) 2024-12-02T14:21:23,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742340_1516 (size=12301) 2024-12-02T14:21:23,901 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/56a8ccf048e649d1bd06a9cdd35d7a32 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/56a8ccf048e649d1bd06a9cdd35d7a32 2024-12-02T14:21:23,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=757 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/8b705214ea5849e0843ae82c289c9dde 2024-12-02T14:21:23,906 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9c93976ab5d5804c690c7e75c9981484/C of 9c93976ab5d5804c690c7e75c9981484 into 56a8ccf048e649d1bd06a9cdd35d7a32(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:23,906 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:23,906 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., storeName=9c93976ab5d5804c690c7e75c9981484/C, priority=13, startTime=1733149283053; duration=0sec 2024-12-02T14:21:23,906 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:23,906 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9c93976ab5d5804c690c7e75c9981484:C 2024-12-02T14:21:23,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/f9bc407343c349b3bfa6a0df00314dd8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/f9bc407343c349b3bfa6a0df00314dd8 2024-12-02T14:21:23,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/f9bc407343c349b3bfa6a0df00314dd8, entries=150, sequenceid=757, filesize=12.0 K 2024-12-02T14:21:23,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/5b0b794c7c9d46ca854ef837e9acdd0b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/5b0b794c7c9d46ca854ef837e9acdd0b 2024-12-02T14:21:23,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/5b0b794c7c9d46ca854ef837e9acdd0b, entries=150, sequenceid=757, filesize=12.0 K 2024-12-02T14:21:23,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/8b705214ea5849e0843ae82c289c9dde as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/8b705214ea5849e0843ae82c289c9dde 2024-12-02T14:21:23,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/8b705214ea5849e0843ae82c289c9dde, entries=150, sequenceid=757, filesize=12.0 K 2024-12-02T14:21:23,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=13.42 KB/13740 for 9c93976ab5d5804c690c7e75c9981484 in 857ms, sequenceid=757, compaction requested=false 2024-12-02T14:21:23,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9c93976ab5d5804c690c7e75c9981484: 2024-12-02T14:21:27,048 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T14:21:32,090 INFO [Thread-1925 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1925. 2024-12-02T14:21:32,090 DEBUG [Thread-1925 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:21:32,090 DEBUG [Thread-1925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:32,090 DEBUG [Thread-1925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:32,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:32,139 INFO [Thread-1923 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1923. 2024-12-02T14:21:32,139 DEBUG [Thread-1923 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:21:32,139 DEBUG [Thread-1923 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:32,139 DEBUG [Thread-1923 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:32,139 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 161 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 222 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 210 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1768 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5299 rows 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1764 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5291 rows 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1767 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5299 rows 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1783 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5346 rows 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1763 2024-12-02T14:21:32,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5287 rows 2024-12-02T14:21:32,140 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T14:21:32,140 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T14:21:32,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:32,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:32,140 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:32,141 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T14:21:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T14:21:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T14:21:32,143 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149292143"}]},"ts":"1733149292143"} 2024-12-02T14:21:32,144 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T14:21:32,144 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T14:21:32,145 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T14:21:32,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c93976ab5d5804c690c7e75c9981484, UNASSIGN}] 2024-12-02T14:21:32,146 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c93976ab5d5804c690c7e75c9981484, UNASSIGN 2024-12-02T14:21:32,147 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=9c93976ab5d5804c690c7e75c9981484, regionState=CLOSING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:32,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c93976ab5d5804c690c7e75c9981484, UNASSIGN because future has completed 2024-12-02T14:21:32,148 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T14:21:32,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:21:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T14:21:32,301 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:32,301 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T14:21:32,301 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 9c93976ab5d5804c690c7e75c9981484, disabling compactions & flushes 2024-12-02T14:21:32,301 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:32,301 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:32,301 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. after waiting 0 ms 2024-12-02T14:21:32,301 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:32,302 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing 9c93976ab5d5804c690c7e75c9981484 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-02T14:21:32,302 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=A 2024-12-02T14:21:32,302 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:32,302 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=B 2024-12-02T14:21:32,302 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:32,302 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9c93976ab5d5804c690c7e75c9981484, store=C 2024-12-02T14:21:32,302 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:32,310 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/06c2f4548eef4f9cacb6eb602dc7278f is 50, key is test_row_0/A:col10/1733149292138/Put/seqid=0 2024-12-02T14:21:32,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742341_1517 (size=9857) 2024-12-02T14:21:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T14:21:32,716 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=767 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/06c2f4548eef4f9cacb6eb602dc7278f 2024-12-02T14:21:32,725 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/bcafb7f48bf241ea95cf930773482a6d is 50, key is test_row_0/B:col10/1733149292138/Put/seqid=0 2024-12-02T14:21:32,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742342_1518 (size=9857) 2024-12-02T14:21:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T14:21:33,135 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=767 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/bcafb7f48bf241ea95cf930773482a6d 2024-12-02T14:21:33,149 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/1114f70f450e4f2db151101314b85bf2 is 50, key is test_row_0/C:col10/1733149292138/Put/seqid=0 2024-12-02T14:21:33,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742343_1519 (size=9857) 2024-12-02T14:21:33,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T14:21:33,554 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=767 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/1114f70f450e4f2db151101314b85bf2 2024-12-02T14:21:33,557 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/A/06c2f4548eef4f9cacb6eb602dc7278f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/06c2f4548eef4f9cacb6eb602dc7278f 2024-12-02T14:21:33,560 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/06c2f4548eef4f9cacb6eb602dc7278f, entries=100, sequenceid=767, filesize=9.6 K 2024-12-02T14:21:33,561 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/B/bcafb7f48bf241ea95cf930773482a6d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/bcafb7f48bf241ea95cf930773482a6d 2024-12-02T14:21:33,564 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/bcafb7f48bf241ea95cf930773482a6d, entries=100, sequenceid=767, filesize=9.6 K 2024-12-02T14:21:33,564 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/.tmp/C/1114f70f450e4f2db151101314b85bf2 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/1114f70f450e4f2db151101314b85bf2 2024-12-02T14:21:33,567 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/1114f70f450e4f2db151101314b85bf2, entries=100, sequenceid=767, filesize=9.6 K 2024-12-02T14:21:33,568 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 9c93976ab5d5804c690c7e75c9981484 in 1266ms, sequenceid=767, compaction requested=true 2024-12-02T14:21:33,569 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/a4c3b68f1114435ba8b6c98a0541b374, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8554089ac7f24b699af9e9bac04cacb0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/9ff48cde35ab4133a06b731b0789a6db, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/e764e2aa12dc49598e35239cd0a43b76, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/120566003b46423d9a45e7edd65d9be3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c6aaa66236a24b869e1cdfba1ca31f1d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/ae25e2bf90564aa6b25c22ff4e36bda0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/424afcdfa49a41f7bb32f917b39805de, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/9b609ba1c48c4940a6817b62a83b7529, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/4ff6c2c4f18247b7bb19b874e29f1edd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/d78001eeea2140d79fb0b2bc3d69b053, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/3bd2ba5962d240e0abbf12110db75987, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/0244e0e233524b9a90ecc236baaca543, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/1e5752a6081f45e7bf83fd051f2c5090, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c38e5f528f564213b081ca9bc77c03b8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8862fd315a5c45bbb89ef4f90d36998c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c2c7c7e42dda415fbe5c90ad67aacc07, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/143895dd57744e09b02d068a07539a77, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8e01c3e170e84d1fb9e8a4e05d0108cc, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/7f34dd9e339249238c5f0e3d9ea078cd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/873ad2aa7aec4cfa8c2277576469f113, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/b7e2c58acede44db90973ca5403f8b41, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/477a07a97fcc4ef3b7aba64204580399, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/312221c0f6a14adb9f5d7eb6d3aaed94, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/70cb2d2e42b94bdfb6aad51d48742421, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/49d25d273dab4f958246dede1b575016, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/801c21498ef049c8a24b10ebea4d0bd8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/fb1d1d4777b0475982e7f8b8cc725e5a] to archive 2024-12-02T14:21:33,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:21:33,571 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/a4c3b68f1114435ba8b6c98a0541b374 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/a4c3b68f1114435ba8b6c98a0541b374 2024-12-02T14:21:33,572 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8554089ac7f24b699af9e9bac04cacb0 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8554089ac7f24b699af9e9bac04cacb0 2024-12-02T14:21:33,573 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/9ff48cde35ab4133a06b731b0789a6db to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/9ff48cde35ab4133a06b731b0789a6db 2024-12-02T14:21:33,574 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/e764e2aa12dc49598e35239cd0a43b76 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/e764e2aa12dc49598e35239cd0a43b76 2024-12-02T14:21:33,575 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/120566003b46423d9a45e7edd65d9be3 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/120566003b46423d9a45e7edd65d9be3 2024-12-02T14:21:33,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c6aaa66236a24b869e1cdfba1ca31f1d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c6aaa66236a24b869e1cdfba1ca31f1d 2024-12-02T14:21:33,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/ae25e2bf90564aa6b25c22ff4e36bda0 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/ae25e2bf90564aa6b25c22ff4e36bda0 2024-12-02T14:21:33,577 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/424afcdfa49a41f7bb32f917b39805de to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/424afcdfa49a41f7bb32f917b39805de 2024-12-02T14:21:33,578 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/9b609ba1c48c4940a6817b62a83b7529 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/9b609ba1c48c4940a6817b62a83b7529 2024-12-02T14:21:33,579 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/4ff6c2c4f18247b7bb19b874e29f1edd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/4ff6c2c4f18247b7bb19b874e29f1edd 2024-12-02T14:21:33,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/d78001eeea2140d79fb0b2bc3d69b053 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/d78001eeea2140d79fb0b2bc3d69b053 2024-12-02T14:21:33,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/3bd2ba5962d240e0abbf12110db75987 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/3bd2ba5962d240e0abbf12110db75987 2024-12-02T14:21:33,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/0244e0e233524b9a90ecc236baaca543 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/0244e0e233524b9a90ecc236baaca543 2024-12-02T14:21:33,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/1e5752a6081f45e7bf83fd051f2c5090 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/1e5752a6081f45e7bf83fd051f2c5090 2024-12-02T14:21:33,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c38e5f528f564213b081ca9bc77c03b8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c38e5f528f564213b081ca9bc77c03b8 2024-12-02T14:21:33,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8862fd315a5c45bbb89ef4f90d36998c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8862fd315a5c45bbb89ef4f90d36998c 2024-12-02T14:21:33,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c2c7c7e42dda415fbe5c90ad67aacc07 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/c2c7c7e42dda415fbe5c90ad67aacc07 2024-12-02T14:21:33,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/143895dd57744e09b02d068a07539a77 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/143895dd57744e09b02d068a07539a77 2024-12-02T14:21:33,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8e01c3e170e84d1fb9e8a4e05d0108cc to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/8e01c3e170e84d1fb9e8a4e05d0108cc 2024-12-02T14:21:33,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/7f34dd9e339249238c5f0e3d9ea078cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/7f34dd9e339249238c5f0e3d9ea078cd 2024-12-02T14:21:33,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/873ad2aa7aec4cfa8c2277576469f113 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/873ad2aa7aec4cfa8c2277576469f113 2024-12-02T14:21:33,590 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/b7e2c58acede44db90973ca5403f8b41 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/b7e2c58acede44db90973ca5403f8b41 2024-12-02T14:21:33,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/477a07a97fcc4ef3b7aba64204580399 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/477a07a97fcc4ef3b7aba64204580399 2024-12-02T14:21:33,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/312221c0f6a14adb9f5d7eb6d3aaed94 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/312221c0f6a14adb9f5d7eb6d3aaed94 2024-12-02T14:21:33,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/70cb2d2e42b94bdfb6aad51d48742421 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/70cb2d2e42b94bdfb6aad51d48742421 2024-12-02T14:21:33,594 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/49d25d273dab4f958246dede1b575016 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/49d25d273dab4f958246dede1b575016 2024-12-02T14:21:33,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/801c21498ef049c8a24b10ebea4d0bd8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/801c21498ef049c8a24b10ebea4d0bd8 2024-12-02T14:21:33,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/fb1d1d4777b0475982e7f8b8cc725e5a to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/fb1d1d4777b0475982e7f8b8cc725e5a 2024-12-02T14:21:33,597 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/9a52f4fbd8cf48e381d0f22738e6c9be, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/3ffbdd34cd82476c9b8f190d7d086255, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/68d4bcf3f4844a8ebccb158c0e6695f7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/ab907ce7b2bd4ff1ae6d413d271dbc1d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/5835ed668c384b6fa0639dfe1feeaf81, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/593eb9e51cac446891d5ec6d0c135f54, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/05a14d855ab844cea9eccbba0eb9b7c0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/07f38a6f44734db8a97c3194158e58e6, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/e007275ab6884d64b71f0aa4d967e3dd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/6cacf0176b6c452c805db0f00353ecda, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/04ac8374a0944755a408e003e57870b7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/71ceba4c482b458888672ee9ab78f48d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/0711e73ab342430aa723494805e245da, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/67e5f2f264af4f1da23c9fbde146cd06, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/81f6363da23941c4b2795dc6551df979, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/69cc0a4e39a442159d97abdd9571aa75, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/5c5435d4f66f4df785fb17109cd9aa18, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/88c462d5fe014246bbaaccc0a4ca9d56, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/540d7d1bce184ec5a9ffab385154cdf6, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/59a947dbbd69491eb1f66f852392ed5f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/acdaa0d320814c63802436f0f679b433, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/f173177a5ddc45a7bad29d1d934689f9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/4734a5f4277a4ec584602c06302a6e28, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/a62b8a12769242119eb320668f3576e8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/edbc767234734e429c4e7038bacc255e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/626e4ffb2a8c4fa6aff601c97a600000, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/3b67dfcb2d1243f697b8c2be46165f7d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/d93c87388ba7480e95b4101524c4e36f] to archive 2024-12-02T14:21:33,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:21:33,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/9a52f4fbd8cf48e381d0f22738e6c9be to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/9a52f4fbd8cf48e381d0f22738e6c9be 2024-12-02T14:21:33,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/3ffbdd34cd82476c9b8f190d7d086255 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/3ffbdd34cd82476c9b8f190d7d086255 2024-12-02T14:21:33,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/68d4bcf3f4844a8ebccb158c0e6695f7 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/68d4bcf3f4844a8ebccb158c0e6695f7 2024-12-02T14:21:33,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/ab907ce7b2bd4ff1ae6d413d271dbc1d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/ab907ce7b2bd4ff1ae6d413d271dbc1d 2024-12-02T14:21:33,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/5835ed668c384b6fa0639dfe1feeaf81 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/5835ed668c384b6fa0639dfe1feeaf81 2024-12-02T14:21:33,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/593eb9e51cac446891d5ec6d0c135f54 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/593eb9e51cac446891d5ec6d0c135f54 2024-12-02T14:21:33,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/05a14d855ab844cea9eccbba0eb9b7c0 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/05a14d855ab844cea9eccbba0eb9b7c0 2024-12-02T14:21:33,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/07f38a6f44734db8a97c3194158e58e6 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/07f38a6f44734db8a97c3194158e58e6 2024-12-02T14:21:33,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/e007275ab6884d64b71f0aa4d967e3dd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/e007275ab6884d64b71f0aa4d967e3dd 2024-12-02T14:21:33,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/6cacf0176b6c452c805db0f00353ecda to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/6cacf0176b6c452c805db0f00353ecda 2024-12-02T14:21:33,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/04ac8374a0944755a408e003e57870b7 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/04ac8374a0944755a408e003e57870b7 2024-12-02T14:21:33,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/71ceba4c482b458888672ee9ab78f48d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/71ceba4c482b458888672ee9ab78f48d 2024-12-02T14:21:33,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/0711e73ab342430aa723494805e245da to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/0711e73ab342430aa723494805e245da 2024-12-02T14:21:33,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/67e5f2f264af4f1da23c9fbde146cd06 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/67e5f2f264af4f1da23c9fbde146cd06 2024-12-02T14:21:33,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/81f6363da23941c4b2795dc6551df979 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/81f6363da23941c4b2795dc6551df979 2024-12-02T14:21:33,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/69cc0a4e39a442159d97abdd9571aa75 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/69cc0a4e39a442159d97abdd9571aa75 2024-12-02T14:21:33,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/5c5435d4f66f4df785fb17109cd9aa18 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/5c5435d4f66f4df785fb17109cd9aa18 2024-12-02T14:21:33,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/88c462d5fe014246bbaaccc0a4ca9d56 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/88c462d5fe014246bbaaccc0a4ca9d56 2024-12-02T14:21:33,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/540d7d1bce184ec5a9ffab385154cdf6 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/540d7d1bce184ec5a9ffab385154cdf6 2024-12-02T14:21:33,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/59a947dbbd69491eb1f66f852392ed5f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/59a947dbbd69491eb1f66f852392ed5f 2024-12-02T14:21:33,625 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/acdaa0d320814c63802436f0f679b433 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/acdaa0d320814c63802436f0f679b433 2024-12-02T14:21:33,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/f173177a5ddc45a7bad29d1d934689f9 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/f173177a5ddc45a7bad29d1d934689f9 2024-12-02T14:21:33,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/4734a5f4277a4ec584602c06302a6e28 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/4734a5f4277a4ec584602c06302a6e28 2024-12-02T14:21:33,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/a62b8a12769242119eb320668f3576e8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/a62b8a12769242119eb320668f3576e8 2024-12-02T14:21:33,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/edbc767234734e429c4e7038bacc255e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/edbc767234734e429c4e7038bacc255e 2024-12-02T14:21:33,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/626e4ffb2a8c4fa6aff601c97a600000 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/626e4ffb2a8c4fa6aff601c97a600000 2024-12-02T14:21:33,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/3b67dfcb2d1243f697b8c2be46165f7d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/3b67dfcb2d1243f697b8c2be46165f7d 2024-12-02T14:21:33,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/d93c87388ba7480e95b4101524c4e36f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/d93c87388ba7480e95b4101524c4e36f 2024-12-02T14:21:33,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/14e573f7e4964c40a27df72513a39f80, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/51a9422b52dd41e2b5802783320367ad, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/d01003f9ca3f43b8aace459342aa70ad, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/8856ab80844c47bb8afe91c92945b1f1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/26bcd0eede984095b705269218b58615, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/bfe245a1b9874e38beae805ef5f20786, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/98019000ff9742dda372f0a34b56c872, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/d9597c9c3da64f3085c2b16423024199, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/7032af9358594d3b917dc1b77774df69, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/a799e6e8c1aa45058233bce73a049505, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/dd47c9bfbf094a2b958ee2a7922849f1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/96b654d2fc594111a874410b90bda2d5, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/89cd22ab716641538ca1790d6d839f4a, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/1bfec387cadf4f64b60e0f024cad75d9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/5a37891eac72493a9538cb59460aaef9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/4e78a3aee7894b9c956e34adb442c71e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/4ccb00362b804ea7b7f0f105efd68873, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/dc3ecb049f6649d8be97b8f568e054e3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/5dd9d850ca814c689b9ae831542867bc, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/ee8880e272fb4459b80af3e52e6db37d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/22fe181c85134d30a0ad0d7d0958834d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/c2ae3d56569c4f688aaf4a44ec85f2c9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/7198d7e88da14187becc08da2600632f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/aab0bcffb50d4393becadc67e3b874d2, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/f702737a2521432c96b8f290737c9603, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/f79b50d636e841af8b7d0f911977809e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/17d4f3644ab94870b0f9d13fe843ac9d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/9577858966524eb39e13044ddb58e84e] to archive 2024-12-02T14:21:33,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:21:33,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/14e573f7e4964c40a27df72513a39f80 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/14e573f7e4964c40a27df72513a39f80 2024-12-02T14:21:33,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/51a9422b52dd41e2b5802783320367ad to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/51a9422b52dd41e2b5802783320367ad 2024-12-02T14:21:33,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/d01003f9ca3f43b8aace459342aa70ad to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/d01003f9ca3f43b8aace459342aa70ad 2024-12-02T14:21:33,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/8856ab80844c47bb8afe91c92945b1f1 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/8856ab80844c47bb8afe91c92945b1f1 2024-12-02T14:21:33,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/26bcd0eede984095b705269218b58615 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/26bcd0eede984095b705269218b58615 2024-12-02T14:21:33,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/bfe245a1b9874e38beae805ef5f20786 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/bfe245a1b9874e38beae805ef5f20786 2024-12-02T14:21:33,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/98019000ff9742dda372f0a34b56c872 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/98019000ff9742dda372f0a34b56c872 2024-12-02T14:21:33,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/d9597c9c3da64f3085c2b16423024199 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/d9597c9c3da64f3085c2b16423024199 2024-12-02T14:21:33,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/7032af9358594d3b917dc1b77774df69 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/7032af9358594d3b917dc1b77774df69 2024-12-02T14:21:33,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/a799e6e8c1aa45058233bce73a049505 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/a799e6e8c1aa45058233bce73a049505 2024-12-02T14:21:33,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/dd47c9bfbf094a2b958ee2a7922849f1 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/dd47c9bfbf094a2b958ee2a7922849f1 2024-12-02T14:21:33,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/96b654d2fc594111a874410b90bda2d5 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/96b654d2fc594111a874410b90bda2d5 2024-12-02T14:21:33,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/89cd22ab716641538ca1790d6d839f4a to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/89cd22ab716641538ca1790d6d839f4a 2024-12-02T14:21:33,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/1bfec387cadf4f64b60e0f024cad75d9 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/1bfec387cadf4f64b60e0f024cad75d9 2024-12-02T14:21:33,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/5a37891eac72493a9538cb59460aaef9 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/5a37891eac72493a9538cb59460aaef9 2024-12-02T14:21:33,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/4e78a3aee7894b9c956e34adb442c71e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/4e78a3aee7894b9c956e34adb442c71e 2024-12-02T14:21:33,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/4ccb00362b804ea7b7f0f105efd68873 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/4ccb00362b804ea7b7f0f105efd68873 2024-12-02T14:21:33,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/dc3ecb049f6649d8be97b8f568e054e3 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/dc3ecb049f6649d8be97b8f568e054e3 2024-12-02T14:21:33,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/5dd9d850ca814c689b9ae831542867bc to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/5dd9d850ca814c689b9ae831542867bc 2024-12-02T14:21:33,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/ee8880e272fb4459b80af3e52e6db37d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/ee8880e272fb4459b80af3e52e6db37d 2024-12-02T14:21:33,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/22fe181c85134d30a0ad0d7d0958834d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/22fe181c85134d30a0ad0d7d0958834d 2024-12-02T14:21:33,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/c2ae3d56569c4f688aaf4a44ec85f2c9 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/c2ae3d56569c4f688aaf4a44ec85f2c9 2024-12-02T14:21:33,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/7198d7e88da14187becc08da2600632f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/7198d7e88da14187becc08da2600632f 2024-12-02T14:21:33,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/aab0bcffb50d4393becadc67e3b874d2 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/aab0bcffb50d4393becadc67e3b874d2 2024-12-02T14:21:33,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/f702737a2521432c96b8f290737c9603 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/f702737a2521432c96b8f290737c9603 2024-12-02T14:21:33,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/f79b50d636e841af8b7d0f911977809e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/f79b50d636e841af8b7d0f911977809e 2024-12-02T14:21:33,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/17d4f3644ab94870b0f9d13fe843ac9d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/17d4f3644ab94870b0f9d13fe843ac9d 2024-12-02T14:21:33,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/9577858966524eb39e13044ddb58e84e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/9577858966524eb39e13044ddb58e84e 2024-12-02T14:21:33,665 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/recovered.edits/770.seqid, newMaxSeqId=770, maxSeqId=1 2024-12-02T14:21:33,665 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. 2024-12-02T14:21:33,665 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 9c93976ab5d5804c690c7e75c9981484: Waiting for close lock at 1733149292301Running coprocessor pre-close hooks at 1733149292301Disabling compacts and flushes for region at 1733149292301Disabling writes for close at 1733149292301Obtaining lock to block concurrent updates at 1733149292302 (+1 ms)Preparing flush snapshotting stores in 9c93976ab5d5804c690c7e75c9981484 at 1733149292302Finished memstore snapshotting TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733149292302Flushing stores of TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484. at 1733149292303 (+1 ms)Flushing 9c93976ab5d5804c690c7e75c9981484/A: creating writer at 1733149292304 (+1 ms)Flushing 9c93976ab5d5804c690c7e75c9981484/A: appending metadata at 1733149292309 (+5 ms)Flushing 9c93976ab5d5804c690c7e75c9981484/A: closing flushed file at 1733149292309Flushing 9c93976ab5d5804c690c7e75c9981484/B: creating writer at 1733149292722 (+413 ms)Flushing 9c93976ab5d5804c690c7e75c9981484/B: appending metadata at 1733149292725 (+3 ms)Flushing 9c93976ab5d5804c690c7e75c9981484/B: closing flushed file at 1733149292725Flushing 9c93976ab5d5804c690c7e75c9981484/C: creating writer at 1733149293143 (+418 ms)Flushing 9c93976ab5d5804c690c7e75c9981484/C: appending metadata at 1733149293149 (+6 ms)Flushing 9c93976ab5d5804c690c7e75c9981484/C: closing flushed file at 1733149293149Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2aae30b: reopening flushed file at 1733149293557 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b007e46: reopening flushed file at 1733149293560 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@492ee9f6: reopening flushed file at 1733149293564 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 9c93976ab5d5804c690c7e75c9981484 in 1266ms, sequenceid=767, compaction requested=true at 1733149293568 (+4 ms)Writing region close event to WAL at 1733149293660 (+92 ms)Running coprocessor post-close hooks at 1733149293665 (+5 ms)Closed at 1733149293665 2024-12-02T14:21:33,667 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:33,668 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=9c93976ab5d5804c690c7e75c9981484, regionState=CLOSED 2024-12-02T14:21:33,669 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:21:33,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-02T14:21:33,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 9c93976ab5d5804c690c7e75c9981484, server=a3a61c9ba14f,46793,1733149138543 in 1.5230 sec 2024-12-02T14:21:33,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-02T14:21:33,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9c93976ab5d5804c690c7e75c9981484, UNASSIGN in 1.5260 sec 2024-12-02T14:21:33,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-02T14:21:33,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5300 sec 2024-12-02T14:21:33,678 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149293678"}]},"ts":"1733149293678"} 2024-12-02T14:21:33,679 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T14:21:33,679 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T14:21:33,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5390 sec 2024-12-02T14:21:34,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T14:21:34,289 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:34,290 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T14:21:34,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:21:34,292 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:21:34,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T14:21:34,293 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:21:34,295 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:34,297 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/recovered.edits] 2024-12-02T14:21:34,299 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/06c2f4548eef4f9cacb6eb602dc7278f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/06c2f4548eef4f9cacb6eb602dc7278f 2024-12-02T14:21:34,300 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/28b20568affc41f790b249a9e91d64cc to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/28b20568affc41f790b249a9e91d64cc 2024-12-02T14:21:34,301 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/f9bc407343c349b3bfa6a0df00314dd8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/A/f9bc407343c349b3bfa6a0df00314dd8 2024-12-02T14:21:34,303 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/4c47bd1072dd4ca6b722afb72f50736a to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/4c47bd1072dd4ca6b722afb72f50736a 2024-12-02T14:21:34,304 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/5b0b794c7c9d46ca854ef837e9acdd0b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/5b0b794c7c9d46ca854ef837e9acdd0b 2024-12-02T14:21:34,306 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/bcafb7f48bf241ea95cf930773482a6d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/B/bcafb7f48bf241ea95cf930773482a6d 2024-12-02T14:21:34,308 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/1114f70f450e4f2db151101314b85bf2 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/1114f70f450e4f2db151101314b85bf2 2024-12-02T14:21:34,309 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/56a8ccf048e649d1bd06a9cdd35d7a32 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/56a8ccf048e649d1bd06a9cdd35d7a32 2024-12-02T14:21:34,310 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/8b705214ea5849e0843ae82c289c9dde to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/C/8b705214ea5849e0843ae82c289c9dde 2024-12-02T14:21:34,312 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/recovered.edits/770.seqid to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484/recovered.edits/770.seqid 2024-12-02T14:21:34,312 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/9c93976ab5d5804c690c7e75c9981484 2024-12-02T14:21:34,313 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T14:21:34,315 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:21:34,317 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T14:21:34,320 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T14:21:34,321 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:21:34,321 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T14:21:34,321 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733149294321"}]},"ts":"9223372036854775807"} 2024-12-02T14:21:34,324 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T14:21:34,324 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9c93976ab5d5804c690c7e75c9981484, NAME => 'TestAcidGuarantees,,1733149259745.9c93976ab5d5804c690c7e75c9981484.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T14:21:34,324 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T14:21:34,324 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733149294324"}]},"ts":"9223372036854775807"} 2024-12-02T14:21:34,327 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T14:21:34,327 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:21:34,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 37 msec 2024-12-02T14:21:34,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T14:21:34,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T14:21:34,399 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:34,410 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=245 (was 250), OpenFileDescriptor=441 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=377 (was 488), ProcessCount=11 (was 11), AvailableMemoryMB=6637 (was 5899) - AvailableMemoryMB LEAK? - 2024-12-02T14:21:34,422 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=245, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=377, ProcessCount=11, AvailableMemoryMB=6628 2024-12-02T14:21:34,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T14:21:34,424 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T14:21:34,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T14:21:34,426 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T14:21:34,426 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:34,426 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-12-02T14:21:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-02T14:21:34,428 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T14:21:34,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742344_1520 (size=1061) 2024-12-02T14:21:34,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-02T14:21:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-02T14:21:34,835 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919 2024-12-02T14:21:34,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742345_1521 (size=53) 2024-12-02T14:21:35,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-02T14:21:35,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:21:35,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1a7e8a8b8e1f978fa1c5a7606b30770c, disabling compactions & flushes 2024-12-02T14:21:35,241 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:35,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:35,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. after waiting 0 ms 2024-12-02T14:21:35,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:35,241 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:35,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: Waiting for close lock at 1733149295241Disabling compacts and flushes for region at 1733149295241Disabling writes for close at 1733149295241Writing region close event to WAL at 1733149295241Closed at 1733149295241 2024-12-02T14:21:35,243 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T14:21:35,243 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733149295243"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733149295243"}]},"ts":"1733149295243"} 2024-12-02T14:21:35,245 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T14:21:35,246 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T14:21:35,246 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149295246"}]},"ts":"1733149295246"} 2024-12-02T14:21:35,248 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T14:21:35,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a7e8a8b8e1f978fa1c5a7606b30770c, ASSIGN}] 2024-12-02T14:21:35,249 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a7e8a8b8e1f978fa1c5a7606b30770c, ASSIGN 2024-12-02T14:21:35,250 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a7e8a8b8e1f978fa1c5a7606b30770c, ASSIGN; state=OFFLINE, location=a3a61c9ba14f,46793,1733149138543; forceNewPlan=false, retain=false 2024-12-02T14:21:35,401 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=1a7e8a8b8e1f978fa1c5a7606b30770c, regionState=OPENING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:35,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a7e8a8b8e1f978fa1c5a7606b30770c, ASSIGN because future has completed 2024-12-02T14:21:35,403 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:21:35,561 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:35,561 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} 2024-12-02T14:21:35,562 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:35,562 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:21:35,562 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:35,562 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:35,563 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:35,565 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:21:35,565 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a7e8a8b8e1f978fa1c5a7606b30770c columnFamilyName A 2024-12-02T14:21:35,565 DEBUG [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:35,565 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.HStore(327): Store=1a7e8a8b8e1f978fa1c5a7606b30770c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:21:35,566 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:35,566 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:21:35,566 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a7e8a8b8e1f978fa1c5a7606b30770c columnFamilyName B 2024-12-02T14:21:35,567 DEBUG [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:35,567 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.HStore(327): Store=1a7e8a8b8e1f978fa1c5a7606b30770c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:21:35,567 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:35,568 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:21:35,568 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a7e8a8b8e1f978fa1c5a7606b30770c columnFamilyName C 2024-12-02T14:21:35,568 DEBUG [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:35,568 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.HStore(327): Store=1a7e8a8b8e1f978fa1c5a7606b30770c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:21:35,568 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:35,569 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-02T14:21:35,569 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:35,570 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:35,570 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:35,570 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:35,571 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T14:21:35,572 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:35,573 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T14:21:35,573 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 1a7e8a8b8e1f978fa1c5a7606b30770c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66957305, jitterRate=-0.002258405089378357}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T14:21:35,574 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:35,574 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: Running coprocessor pre-open hook at 1733149295562Writing region info on filesystem at 1733149295562Initializing all the Stores at 1733149295563 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149295563Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149295563Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149295563Cleaning up temporary data from old regions at 1733149295570 (+7 ms)Running coprocessor post-open hooks at 1733149295574 (+4 ms)Region opened successfully at 1733149295574 2024-12-02T14:21:35,575 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., pid=140, masterSystemTime=1733149295554 2024-12-02T14:21:35,576 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:35,576 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:35,577 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=1a7e8a8b8e1f978fa1c5a7606b30770c, regionState=OPEN, openSeqNum=2, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:35,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:21:35,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-02T14:21:35,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 in 177 msec 2024-12-02T14:21:35,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-02T14:21:35,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a7e8a8b8e1f978fa1c5a7606b30770c, ASSIGN in 334 msec 2024-12-02T14:21:35,584 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T14:21:35,584 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149295584"}]},"ts":"1733149295584"} 2024-12-02T14:21:35,585 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T14:21:35,586 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T14:21:35,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-12-02T14:21:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-02T14:21:36,579 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:36,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695f7e1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:36,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:36,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:36,581 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22da96d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:36,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:36,583 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:36,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ce1808, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:36,584 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:36,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:36,586 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T14:21:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is a3a61c9ba14f,35877,1733149137941 2024-12-02T14:21:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@28e9fa24 2024-12-02T14:21:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T14:21:36,588 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T14:21:36,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T14:21:36,590 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T14:21:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T14:21:36,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742346_1522 (size=1097) 2024-12-02T14:21:37,006 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T14:21:37,006 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T14:21:37,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T14:21:37,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a7e8a8b8e1f978fa1c5a7606b30770c, REOPEN/MOVE}] 2024-12-02T14:21:37,015 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a7e8a8b8e1f978fa1c5a7606b30770c, REOPEN/MOVE 2024-12-02T14:21:37,016 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1a7e8a8b8e1f978fa1c5a7606b30770c, regionState=CLOSING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a7e8a8b8e1f978fa1c5a7606b30770c, REOPEN/MOVE because future has completed 2024-12-02T14:21:37,017 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T14:21:37,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:21:37,171 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,171 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T14:21:37,171 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 1a7e8a8b8e1f978fa1c5a7606b30770c, disabling compactions & flushes 2024-12-02T14:21:37,171 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:37,171 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:37,171 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. after waiting 0 ms 2024-12-02T14:21:37,171 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:37,179 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T14:21:37,180 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:37,180 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: Waiting for close lock at 1733149297171Running coprocessor pre-close hooks at 1733149297171Disabling compacts and flushes for region at 1733149297171Disabling writes for close at 1733149297171Writing region close event to WAL at 1733149297172 (+1 ms)Running coprocessor post-close hooks at 1733149297180 (+8 ms)Closed at 1733149297180 2024-12-02T14:21:37,180 WARN [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 1a7e8a8b8e1f978fa1c5a7606b30770c to self. 2024-12-02T14:21:37,182 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,183 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1a7e8a8b8e1f978fa1c5a7606b30770c, regionState=CLOSED 2024-12-02T14:21:37,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:21:37,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-02T14:21:37,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 in 168 msec 2024-12-02T14:21:37,188 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a7e8a8b8e1f978fa1c5a7606b30770c, REOPEN/MOVE; state=CLOSED, location=a3a61c9ba14f,46793,1733149138543; forceNewPlan=false, retain=true 2024-12-02T14:21:37,339 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1a7e8a8b8e1f978fa1c5a7606b30770c, regionState=OPENING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a7e8a8b8e1f978fa1c5a7606b30770c, REOPEN/MOVE because future has completed 2024-12-02T14:21:37,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:21:37,494 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:37,495 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} 2024-12-02T14:21:37,495 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,495 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:21:37,495 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,495 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,496 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,496 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:21:37,496 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a7e8a8b8e1f978fa1c5a7606b30770c columnFamilyName A 2024-12-02T14:21:37,497 DEBUG [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:37,498 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.HStore(327): Store=1a7e8a8b8e1f978fa1c5a7606b30770c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:21:37,498 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,498 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:21:37,499 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a7e8a8b8e1f978fa1c5a7606b30770c columnFamilyName B 2024-12-02T14:21:37,499 DEBUG [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:37,499 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.HStore(327): Store=1a7e8a8b8e1f978fa1c5a7606b30770c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:21:37,499 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,499 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T14:21:37,499 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a7e8a8b8e1f978fa1c5a7606b30770c columnFamilyName C 2024-12-02T14:21:37,500 DEBUG [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:37,500 INFO [StoreOpener-1a7e8a8b8e1f978fa1c5a7606b30770c-1 {}] regionserver.HStore(327): Store=1a7e8a8b8e1f978fa1c5a7606b30770c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:21:37,500 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:37,500 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,501 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,501 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,502 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,502 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,502 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T14:21:37,503 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,504 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 1a7e8a8b8e1f978fa1c5a7606b30770c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73373729, jitterRate=0.0933537632226944}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T14:21:37,504 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,504 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: Running coprocessor pre-open hook at 1733149297495Writing region info on filesystem at 1733149297495Initializing all the Stores at 1733149297495Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149297496 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149297496Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149297496Cleaning up temporary data from old regions at 1733149297502 (+6 ms)Running coprocessor post-open hooks at 1733149297504 (+2 ms)Region opened successfully at 1733149297504 2024-12-02T14:21:37,505 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., pid=145, masterSystemTime=1733149297492 2024-12-02T14:21:37,506 DEBUG [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:37,506 INFO [RS_OPEN_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:37,507 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1a7e8a8b8e1f978fa1c5a7606b30770c, regionState=OPEN, openSeqNum=5, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:21:37,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-12-02T14:21:37,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 in 168 msec 2024-12-02T14:21:37,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-02T14:21:37,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a7e8a8b8e1f978fa1c5a7606b30770c, REOPEN/MOVE in 496 msec 2024-12-02T14:21:37,513 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-02T14:21:37,514 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 502 msec 2024-12-02T14:21:37,515 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 924 msec 2024-12-02T14:21:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T14:21:37,516 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:37,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6919536d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,517 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:37,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:37,517 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:37,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@592f139c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:37,518 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37539d5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d1887e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,519 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:37,519 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:37,520 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138e9486, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:37,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:37,521 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:37,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e0c89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e2a1e31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,522 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:37,522 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:37,523 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:37,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:37,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:37,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b662a3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:37,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:37,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:37,525 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:37,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e2f5def, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdeed38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,526 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:37,526 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:37,526 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:37,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:37,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:37,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a9c3fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:37,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:37,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:37,527 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:37,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c254c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b6eabbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,528 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:37,528 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:37,529 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577bdd1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:37,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:37,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:37,531 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:37,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2cfad9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,532 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b5dc35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,532 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:37,532 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:37,532 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:37,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:37,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:37,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a261448, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:37,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:37,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:37,534 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a17fe37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18d7368f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:37,535 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:37,535 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:37,536 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:37,536 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:37,536 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1297acbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,536 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:37,536 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:37,536 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:37,537 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13922282, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108b0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,538 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:37,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:37,539 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:37,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:37,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:37,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5262b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:37,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:37,540 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad98829, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a5be7c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,542 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:37,542 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:37,543 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:37,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:37,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:37,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49ace172, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:37,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:37,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:37,544 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:37,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@198c4a80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@162f1ee9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,545 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,35877,-1 for getting cluster id 2024-12-02T14:21:37,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:21:37,546 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26e152f2-8fae-48f3-8f83-0754b5a7150f' 2024-12-02T14:21:37,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:21:37,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26e152f2-8fae-48f3-8f83-0754b5a7150f" 2024-12-02T14:21:37,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a30119, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,35877,-1] 2024-12-02T14:21:37,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:21:37,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:37,547 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:21:37,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20bc2374, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:21:37,549 DEBUG [Thread-2337 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:37,549 DEBUG [Thread-2341 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:37,549 DEBUG [Thread-2346 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:37,550 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:37,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:37,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:21:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:21:37,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:21:37,557 DEBUG [Thread-2335 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:37,557 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:21:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:37,558 DEBUG [Thread-2339 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-02T14:21:37,559 DEBUG [Thread-2348 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:37,560 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:37,560 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:21:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-02T14:21:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:37,560 DEBUG [Thread-2354 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T14:21:37,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:37,562 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:21:37,562 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:21:37,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:21:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:37,562 DEBUG [Thread-2343 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:21:37,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:37,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:37,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:37,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:37,563 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:37,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:37,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:37,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:37,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:21:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:21:37,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:37,570 DEBUG [Thread-2350 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:37,570 DEBUG [Thread-2352 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:21:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:21:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46793,1733149138543, seqNum=-1] 2024-12-02T14:21:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:21:37,572 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:37,572 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:21:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:21:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5] 2024-12-02T14:21:37,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149297569, max timestamp=1733149297589Num uniques 859; 2024-12-02T14:21:37,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733149297569, max timestamp=1733149297589Num uniques 859; 2024-12-02T14:21:37,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:21:37,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T14:21:37,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733149297581, max timestamp=1733149297589Num uniques 859; 2024-12-02T14:21:37,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3a61c9ba14f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T14:21:37,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202354513869e204aed8f2b7044bafc2a19_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149297555/Put/seqid=0 2024-12-02T14:21:37,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742347_1523 (size=12154) 2024-12-02T14:21:37,617 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:37,621 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202354513869e204aed8f2b7044bafc2a19_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202354513869e204aed8f2b7044bafc2a19_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:37,622 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/221b04bd07ff45399afab9e9fe2d2055, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:37,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/221b04bd07ff45399afab9e9fe2d2055 is 175, key is test_row_0/A:col10/1733149297555/Put/seqid=0 2024-12-02T14:21:37,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:37,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149357620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:37,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149357621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:37,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:37,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:37,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149357625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:37,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149357626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149357626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:37,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742348_1524 (size=30955) 2024-12-02T14:21:37,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-02T14:21:37,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-02T14:21:37,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:37,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:37,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:37,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:37,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:37,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:37,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149357739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149357739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149357739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149357739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:37,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149357739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:37,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-02T14:21:37,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:37,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:37,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:37,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:37,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-02T14:21:37,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:37,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149357949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149357948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:37,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149357949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149357949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:37,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149357949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:38,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-02T14:21:38,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:38,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:38,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:38,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:38,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:38,065 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/221b04bd07ff45399afab9e9fe2d2055 2024-12-02T14:21:38,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/4f554487536a4a3ca5c094b4b7f5242f is 50, key is test_row_0/B:col10/1733149297555/Put/seqid=0 2024-12-02T14:21:38,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742349_1525 (size=12001) 2024-12-02T14:21:38,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/4f554487536a4a3ca5c094b4b7f5242f 2024-12-02T14:21:38,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/9302fc94d21844cc9cdca423e7cd7c96 is 50, key is test_row_0/C:col10/1733149297555/Put/seqid=0 2024-12-02T14:21:38,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742350_1526 (size=12001) 2024-12-02T14:21:38,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-02T14:21:38,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:38,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:38,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:38,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:38,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-02T14:21:38,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149358259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:38,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149358259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:38,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149358259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149358259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149358259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:38,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-02T14:21:38,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:38,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:38,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:38,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:38,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:38,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-02T14:21:38,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:38,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:38,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:38,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:38,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:38,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/9302fc94d21844cc9cdca423e7cd7c96 2024-12-02T14:21:38,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/221b04bd07ff45399afab9e9fe2d2055 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/221b04bd07ff45399afab9e9fe2d2055 2024-12-02T14:21:38,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/221b04bd07ff45399afab9e9fe2d2055, entries=150, sequenceid=15, filesize=30.2 K 2024-12-02T14:21:38,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/4f554487536a4a3ca5c094b4b7f5242f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/4f554487536a4a3ca5c094b4b7f5242f 2024-12-02T14:21:38,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/4f554487536a4a3ca5c094b4b7f5242f, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T14:21:38,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/9302fc94d21844cc9cdca423e7cd7c96 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/9302fc94d21844cc9cdca423e7cd7c96 2024-12-02T14:21:38,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/9302fc94d21844cc9cdca423e7cd7c96, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T14:21:38,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=124.84 KB/127835 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 1028ms, sequenceid=15, compaction requested=false 2024-12-02T14:21:38,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:38,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-02T14:21:38,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:38,635 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=124.84 KB heapSize=354.38 KB 2024-12-02T14:21:38,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:38,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:21:38,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:38,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:21:38,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:38,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T14:21:38,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a27ffe52392b4fb7914dc7bf860b9e2e_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149297624/Put/seqid=0 2024-12-02T14:21:38,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742351_1527 (size=12154) 2024-12-02T14:21:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-02T14:21:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:38,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:38,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149358775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:38,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149358778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:38,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149358780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:38,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149358781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:38,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149358782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:38,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:38,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149358889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149358889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:38,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149358889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:38,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149358890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:38,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149358898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:39,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149359098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:39,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:39,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149359098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:39,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:39,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149359098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:39,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:39,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:39,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149359098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:39,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:39,103 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a27ffe52392b4fb7914dc7bf860b9e2e_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a27ffe52392b4fb7914dc7bf860b9e2e_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:39,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/06b0c8bd65f8428683d2679321c0dc3e, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:39,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/06b0c8bd65f8428683d2679321c0dc3e is 175, key is test_row_0/A:col10/1733149297624/Put/seqid=0 2024-12-02T14:21:39,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742352_1528 (size=30955) 2024-12-02T14:21:39,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149359108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:39,199 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T14:21:39,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149359408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:39,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149359409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149359409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:39,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149359409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:39,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149359418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:39,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:39,508 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/06b0c8bd65f8428683d2679321c0dc3e 2024-12-02T14:21:39,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/256934c23da04e74a68087073c5ef600 is 50, key is test_row_0/B:col10/1733149297624/Put/seqid=0 2024-12-02T14:21:39,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742353_1529 (size=12001) 2024-12-02T14:21:39,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-02T14:21:39,917 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/256934c23da04e74a68087073c5ef600 2024-12-02T14:21:39,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149359918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:39,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149359918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:39,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149359918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149359918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:39,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:39,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/31948a4abfc24b00ac9657bdd85a38a9 is 50, key is test_row_0/C:col10/1733149297624/Put/seqid=0 2024-12-02T14:21:39,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742354_1530 (size=12001) 2024-12-02T14:21:39,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149359928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:39,932 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,932 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:39,932 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:40,326 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/31948a4abfc24b00ac9657bdd85a38a9 2024-12-02T14:21:40,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/06b0c8bd65f8428683d2679321c0dc3e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/06b0c8bd65f8428683d2679321c0dc3e 2024-12-02T14:21:40,332 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/06b0c8bd65f8428683d2679321c0dc3e, entries=150, sequenceid=50, filesize=30.2 K 2024-12-02T14:21:40,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/256934c23da04e74a68087073c5ef600 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/256934c23da04e74a68087073c5ef600 2024-12-02T14:21:40,335 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/256934c23da04e74a68087073c5ef600, entries=150, sequenceid=50, filesize=11.7 K 2024-12-02T14:21:40,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/31948a4abfc24b00ac9657bdd85a38a9 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/31948a4abfc24b00ac9657bdd85a38a9 2024-12-02T14:21:40,338 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/31948a4abfc24b00ac9657bdd85a38a9, entries=150, sequenceid=50, filesize=11.7 K 2024-12-02T14:21:40,339 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~124.84 KB/127835, heapSize ~354.34 KB/362840, currentSize=60.38 KB/61830 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 1705ms, sequenceid=50, compaction requested=false 2024-12-02T14:21:40,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:40,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:40,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-02T14:21:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-02T14:21:40,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-02T14:21:40,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7780 sec 2024-12-02T14:21:40,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.7850 sec 2024-12-02T14:21:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:40,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T14:21:40,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:40,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:40,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:40,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:40,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:40,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:40,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202350c3056624c4f75ac196ea883fd939c_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149300930/Put/seqid=0 2024-12-02T14:21:40,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742355_1531 (size=14594) 2024-12-02T14:21:40,941 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:40,944 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202350c3056624c4f75ac196ea883fd939c_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202350c3056624c4f75ac196ea883fd939c_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:40,945 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/8ab458bcf33048e0b579113acfb68e1c, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:40,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/8ab458bcf33048e0b579113acfb68e1c is 175, key is test_row_0/A:col10/1733149300930/Put/seqid=0 2024-12-02T14:21:40,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149360946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:40,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742356_1532 (size=39549) 2024-12-02T14:21:40,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:40,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:40,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149360949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:40,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:40,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:40,951 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=65, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/8ab458bcf33048e0b579113acfb68e1c 2024-12-02T14:21:40,951 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_1' in 1a7e8a8b8e1f978fa1c5a7606b30770c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:40,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:40,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:40,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:40,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149360949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:40,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149360951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:40,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149360951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:40,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/ef47df58af8b43cb9555f44650901501 is 50, key is test_row_0/B:col10/1733149300930/Put/seqid=0 2024-12-02T14:21:40,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742357_1533 (size=12001) 2024-12-02T14:21:41,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149361059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:41,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149361059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149361059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:41,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149361059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:41,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:41,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149361269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:41,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149361269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:41,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149361269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:41,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149361269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:41,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/ef47df58af8b43cb9555f44650901501 2024-12-02T14:21:41,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/6d8f311baab241a2b3298bf7706c8b19 is 50, key is test_row_0/C:col10/1733149300930/Put/seqid=0 2024-12-02T14:21:41,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742358_1534 (size=12001) 2024-12-02T14:21:41,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:41,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149361578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149361578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:41,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:41,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149361578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149361578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-02T14:21:41,729 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:41,729 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:21:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-02T14:21:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-02T14:21:41,731 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:21:41,731 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:21:41,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:21:41,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/6d8f311baab241a2b3298bf7706c8b19 2024-12-02T14:21:41,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/8ab458bcf33048e0b579113acfb68e1c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8ab458bcf33048e0b579113acfb68e1c 2024-12-02T14:21:41,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8ab458bcf33048e0b579113acfb68e1c, entries=200, sequenceid=65, filesize=38.6 K 2024-12-02T14:21:41,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/ef47df58af8b43cb9555f44650901501 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/ef47df58af8b43cb9555f44650901501 2024-12-02T14:21:41,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/ef47df58af8b43cb9555f44650901501, entries=150, sequenceid=65, filesize=11.7 K 2024-12-02T14:21:41,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/6d8f311baab241a2b3298bf7706c8b19 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/6d8f311baab241a2b3298bf7706c8b19 2024-12-02T14:21:41,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/6d8f311baab241a2b3298bf7706c8b19, entries=150, sequenceid=65, filesize=11.7 K 2024-12-02T14:21:41,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 864ms, sequenceid=65, compaction requested=true 2024-12-02T14:21:41,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:41,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:21:41,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:41,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:21:41,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:41,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:21:41,796 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:41,796 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:41,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:41,796 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:41,796 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:41,796 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/A is initiating minor compaction (all files) 2024-12-02T14:21:41,796 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/B is initiating minor compaction (all files) 2024-12-02T14:21:41,796 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/A in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:41,796 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/B in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:41,796 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/221b04bd07ff45399afab9e9fe2d2055, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/06b0c8bd65f8428683d2679321c0dc3e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8ab458bcf33048e0b579113acfb68e1c] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=99.1 K 2024-12-02T14:21:41,796 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/4f554487536a4a3ca5c094b4b7f5242f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/256934c23da04e74a68087073c5ef600, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/ef47df58af8b43cb9555f44650901501] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=35.2 K 2024-12-02T14:21:41,796 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:41,796 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/221b04bd07ff45399afab9e9fe2d2055, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/06b0c8bd65f8428683d2679321c0dc3e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8ab458bcf33048e0b579113acfb68e1c] 2024-12-02T14:21:41,797 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 221b04bd07ff45399afab9e9fe2d2055, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733149297555 2024-12-02T14:21:41,797 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f554487536a4a3ca5c094b4b7f5242f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733149297555 2024-12-02T14:21:41,797 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 256934c23da04e74a68087073c5ef600, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733149297609 2024-12-02T14:21:41,797 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06b0c8bd65f8428683d2679321c0dc3e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733149297609 2024-12-02T14:21:41,797 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting ef47df58af8b43cb9555f44650901501, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733149300930 2024-12-02T14:21:41,797 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ab458bcf33048e0b579113acfb68e1c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733149298777 2024-12-02T14:21:41,802 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:41,803 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#B#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:41,803 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412025ca586f698a2439483d92fcefe89afe2_1a7e8a8b8e1f978fa1c5a7606b30770c store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:41,803 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/cc81053ee1b0446cb490bd08b3fb6526 is 50, key is test_row_0/B:col10/1733149300930/Put/seqid=0 2024-12-02T14:21:41,805 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412025ca586f698a2439483d92fcefe89afe2_1a7e8a8b8e1f978fa1c5a7606b30770c, store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:41,806 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025ca586f698a2439483d92fcefe89afe2_1a7e8a8b8e1f978fa1c5a7606b30770c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:41,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742359_1535 (size=12104) 2024-12-02T14:21:41,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742360_1536 (size=4469) 2024-12-02T14:21:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-02T14:21:41,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-02T14:21:41,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:41,884 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T14:21:41,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:41,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:41,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:41,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:41,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:41,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:41,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020438354ba5dc447dbff9c18cb69b0fb9_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149300947/Put/seqid=0 2024-12-02T14:21:41,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742361_1537 (size=12154) 2024-12-02T14:21:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-02T14:21:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:42,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:42,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149362101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149362104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149362105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149362106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149362208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,211 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#A#compaction#454 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:42,211 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/cc81053ee1b0446cb490bd08b3fb6526 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/cc81053ee1b0446cb490bd08b3fb6526 2024-12-02T14:21:42,211 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/13cc40aa13264fdc96491b39c407bb24 is 175, key is test_row_0/A:col10/1733149300930/Put/seqid=0 2024-12-02T14:21:42,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742362_1538 (size=31058) 2024-12-02T14:21:42,216 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/B of 1a7e8a8b8e1f978fa1c5a7606b30770c into cc81053ee1b0446cb490bd08b3fb6526(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:42,216 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:42,216 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/B, priority=13, startTime=1733149301796; duration=0sec 2024-12-02T14:21:42,216 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:42,217 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:B 2024-12-02T14:21:42,217 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:42,218 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:42,218 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/C is initiating minor compaction (all files) 2024-12-02T14:21:42,218 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/C in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:42,218 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/9302fc94d21844cc9cdca423e7cd7c96, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/31948a4abfc24b00ac9657bdd85a38a9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/6d8f311baab241a2b3298bf7706c8b19] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=35.2 K 2024-12-02T14:21:42,218 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 9302fc94d21844cc9cdca423e7cd7c96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733149297555 2024-12-02T14:21:42,219 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 31948a4abfc24b00ac9657bdd85a38a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733149297609 2024-12-02T14:21:42,219 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d8f311baab241a2b3298bf7706c8b19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733149300930 2024-12-02T14:21:42,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149362218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149362219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149362219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,225 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#C#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:42,225 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/9f833d8de7624989bd9a1d87628d1ca3 is 50, key is test_row_0/C:col10/1733149300930/Put/seqid=0 2024-12-02T14:21:42,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742363_1539 (size=12104) 2024-12-02T14:21:42,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:42,300 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020438354ba5dc447dbff9c18cb69b0fb9_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020438354ba5dc447dbff9c18cb69b0fb9_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:42,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/3a597ce206784e44b9a6f1fcfc9e4108, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:42,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/3a597ce206784e44b9a6f1fcfc9e4108 is 175, key is test_row_0/A:col10/1733149300947/Put/seqid=0 2024-12-02T14:21:42,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742364_1540 (size=30955) 2024-12-02T14:21:42,306 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/3a597ce206784e44b9a6f1fcfc9e4108 2024-12-02T14:21:42,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/ae75ed02155642a98557033d1b5432c2 is 50, key is test_row_0/B:col10/1733149300947/Put/seqid=0 2024-12-02T14:21:42,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742365_1541 (size=12001) 2024-12-02T14:21:42,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-02T14:21:42,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149362418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149362430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149362430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149362430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,620 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/13cc40aa13264fdc96491b39c407bb24 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/13cc40aa13264fdc96491b39c407bb24 2024-12-02T14:21:42,623 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/A of 1a7e8a8b8e1f978fa1c5a7606b30770c into 13cc40aa13264fdc96491b39c407bb24(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:42,623 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:42,623 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/A, priority=13, startTime=1733149301795; duration=0sec 2024-12-02T14:21:42,623 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:42,623 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:A 2024-12-02T14:21:42,631 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/9f833d8de7624989bd9a1d87628d1ca3 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/9f833d8de7624989bd9a1d87628d1ca3 2024-12-02T14:21:42,635 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/C of 1a7e8a8b8e1f978fa1c5a7606b30770c into 9f833d8de7624989bd9a1d87628d1ca3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:42,635 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:42,635 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/C, priority=13, startTime=1733149301796; duration=0sec 2024-12-02T14:21:42,635 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:42,635 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:C 2024-12-02T14:21:42,715 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/ae75ed02155642a98557033d1b5432c2 2024-12-02T14:21:42,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/3c375ef711e84755a4ac8fd8ccf434fc is 50, key is test_row_0/C:col10/1733149300947/Put/seqid=0 2024-12-02T14:21:42,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742366_1542 (size=12001) 2024-12-02T14:21:42,724 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/3c375ef711e84755a4ac8fd8ccf434fc 2024-12-02T14:21:42,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/3a597ce206784e44b9a6f1fcfc9e4108 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3a597ce206784e44b9a6f1fcfc9e4108 2024-12-02T14:21:42,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149362728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,730 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3a597ce206784e44b9a6f1fcfc9e4108, entries=150, sequenceid=82, filesize=30.2 K 2024-12-02T14:21:42,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/ae75ed02155642a98557033d1b5432c2 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/ae75ed02155642a98557033d1b5432c2 2024-12-02T14:21:42,734 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/ae75ed02155642a98557033d1b5432c2, entries=150, sequenceid=82, filesize=11.7 K 2024-12-02T14:21:42,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/3c375ef711e84755a4ac8fd8ccf434fc as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/3c375ef711e84755a4ac8fd8ccf434fc 2024-12-02T14:21:42,737 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/3c375ef711e84755a4ac8fd8ccf434fc, entries=150, sequenceid=82, filesize=11.7 K 2024-12-02T14:21:42,738 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 854ms, sequenceid=82, compaction requested=false 2024-12-02T14:21:42,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:42,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:42,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-02T14:21:42,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-02T14:21:42,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:42,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T14:21:42,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:42,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:42,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:42,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:42,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:42,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:42,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-02T14:21:42,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0070 sec 2024-12-02T14:21:42,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.0120 sec 2024-12-02T14:21:42,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027afd0e5d20084781a8f3027932cf18b1_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149302096/Put/seqid=0 2024-12-02T14:21:42,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742367_1543 (size=12154) 2024-12-02T14:21:42,749 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:42,753 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027afd0e5d20084781a8f3027932cf18b1_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027afd0e5d20084781a8f3027932cf18b1_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:42,754 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/0bbd94392f6144b184980fee4af8fc33, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:42,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/0bbd94392f6144b184980fee4af8fc33 is 175, key is test_row_0/A:col10/1733149302096/Put/seqid=0 2024-12-02T14:21:42,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149362758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149362759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742368_1544 (size=30955) 2024-12-02T14:21:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,764 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/0bbd94392f6144b184980fee4af8fc33 2024-12-02T14:21:42,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149362762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/25ca66f01fd242e595d1ccd84d6cbf69 is 50, key is test_row_0/B:col10/1733149302096/Put/seqid=0 2024-12-02T14:21:42,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742369_1545 (size=12001) 2024-12-02T14:21:42,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/25ca66f01fd242e595d1ccd84d6cbf69 2024-12-02T14:21:42,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/d6901758c80c4192a8459d759a52a4fe is 50, key is test_row_0/C:col10/1733149302096/Put/seqid=0 2024-12-02T14:21:42,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742370_1546 (size=12001) 2024-12-02T14:21:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-02T14:21:42,869 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:42,869 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:21:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-02T14:21:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T14:21:42,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,870 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:21:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149362868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149362868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,871 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:21:42,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:21:42,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149362879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149362958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:42,961 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_1' in 1a7e8a8b8e1f978fa1c5a7606b30770c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:42,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:42,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T14:21:43,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:43,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:43,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149363078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:43,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149363078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:43,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149363089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:43,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:43,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:43,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T14:21:43,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/d6901758c80c4192a8459d759a52a4fe 2024-12-02T14:21:43,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/0bbd94392f6144b184980fee4af8fc33 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/0bbd94392f6144b184980fee4af8fc33 2024-12-02T14:21:43,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/0bbd94392f6144b184980fee4af8fc33, entries=150, sequenceid=101, filesize=30.2 K 2024-12-02T14:21:43,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/25ca66f01fd242e595d1ccd84d6cbf69 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/25ca66f01fd242e595d1ccd84d6cbf69 2024-12-02T14:21:43,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/25ca66f01fd242e595d1ccd84d6cbf69, entries=150, sequenceid=101, filesize=11.7 K 2024-12-02T14:21:43,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/d6901758c80c4192a8459d759a52a4fe as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/d6901758c80c4192a8459d759a52a4fe 2024-12-02T14:21:43,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/d6901758c80c4192a8459d759a52a4fe, entries=150, sequenceid=101, filesize=11.7 K 2024-12-02T14:21:43,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 467ms, sequenceid=101, compaction requested=true 2024-12-02T14:21:43,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:43,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:21:43,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:43,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:21:43,207 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:43,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:43,208 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:43,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:21:43,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:43,208 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:43,208 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:43,208 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/B is initiating minor compaction (all files) 2024-12-02T14:21:43,208 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/A is initiating minor compaction (all files) 2024-12-02T14:21:43,208 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/B in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,208 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/A in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,209 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/cc81053ee1b0446cb490bd08b3fb6526, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/ae75ed02155642a98557033d1b5432c2, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/25ca66f01fd242e595d1ccd84d6cbf69] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=35.3 K 2024-12-02T14:21:43,209 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/13cc40aa13264fdc96491b39c407bb24, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3a597ce206784e44b9a6f1fcfc9e4108, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/0bbd94392f6144b184980fee4af8fc33] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=90.8 K 2024-12-02T14:21:43,209 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,209 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/13cc40aa13264fdc96491b39c407bb24, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3a597ce206784e44b9a6f1fcfc9e4108, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/0bbd94392f6144b184980fee4af8fc33] 2024-12-02T14:21:43,209 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting cc81053ee1b0446cb490bd08b3fb6526, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733149300930 2024-12-02T14:21:43,209 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13cc40aa13264fdc96491b39c407bb24, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733149300930 2024-12-02T14:21:43,210 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting ae75ed02155642a98557033d1b5432c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733149300945 2024-12-02T14:21:43,210 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a597ce206784e44b9a6f1fcfc9e4108, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733149300945 2024-12-02T14:21:43,210 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 25ca66f01fd242e595d1ccd84d6cbf69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733149302096 2024-12-02T14:21:43,210 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bbd94392f6144b184980fee4af8fc33, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733149302096 2024-12-02T14:21:43,215 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:43,217 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#B#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:43,217 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/7e1e17b740a5435687479fd683358a25 is 50, key is test_row_0/B:col10/1733149302096/Put/seqid=0 2024-12-02T14:21:43,217 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202ae45dce8e1434a1f9ceddcb6c2ed8317_1a7e8a8b8e1f978fa1c5a7606b30770c store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:43,219 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202ae45dce8e1434a1f9ceddcb6c2ed8317_1a7e8a8b8e1f978fa1c5a7606b30770c, store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:43,219 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ae45dce8e1434a1f9ceddcb6c2ed8317_1a7e8a8b8e1f978fa1c5a7606b30770c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:43,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742371_1547 (size=12207) 2024-12-02T14:21:43,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742372_1548 (size=4469) 2024-12-02T14:21:43,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:43,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T14:21:43,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:43,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:43,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:43,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:43,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:43,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:43,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024821679a86974334806faa7e820dcec3_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149303239/Put/seqid=0 2024-12-02T14:21:43,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742373_1549 (size=14594) 2024-12-02T14:21:43,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149363283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:43,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:43,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:43,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149363388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149363388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:43,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149363388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:43,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149363399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:43,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:43,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:43,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T14:21:43,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149363598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:43,625 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#A#compaction#462 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:43,625 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/7e1e17b740a5435687479fd683358a25 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7e1e17b740a5435687479fd683358a25 2024-12-02T14:21:43,625 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/cb03941b1fff47419eef69a868b92889 is 175, key is test_row_0/A:col10/1733149302096/Put/seqid=0 2024-12-02T14:21:43,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742374_1550 (size=31161) 2024-12-02T14:21:43,629 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/B of 1a7e8a8b8e1f978fa1c5a7606b30770c into 7e1e17b740a5435687479fd683358a25(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:43,629 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:43,629 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/B, priority=13, startTime=1733149303207; duration=0sec 2024-12-02T14:21:43,629 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:43,629 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:B 2024-12-02T14:21:43,629 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:43,630 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:43,630 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/C is initiating minor compaction (all files) 2024-12-02T14:21:43,630 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/C in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,630 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/9f833d8de7624989bd9a1d87628d1ca3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/3c375ef711e84755a4ac8fd8ccf434fc, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/d6901758c80c4192a8459d759a52a4fe] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=35.3 K 2024-12-02T14:21:43,630 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f833d8de7624989bd9a1d87628d1ca3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733149300930 2024-12-02T14:21:43,631 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c375ef711e84755a4ac8fd8ccf434fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733149300945 2024-12-02T14:21:43,631 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting d6901758c80c4192a8459d759a52a4fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733149302096 2024-12-02T14:21:43,632 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/cb03941b1fff47419eef69a868b92889 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/cb03941b1fff47419eef69a868b92889 2024-12-02T14:21:43,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:43,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:43,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,635 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/A of 1a7e8a8b8e1f978fa1c5a7606b30770c into cb03941b1fff47419eef69a868b92889(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:43,635 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:43,636 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/A, priority=13, startTime=1733149303207; duration=0sec 2024-12-02T14:21:43,636 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:43,636 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:A 2024-12-02T14:21:43,637 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#C#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:43,637 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/b765e9c08d0c481190e3595ff0059a3b is 50, key is test_row_0/C:col10/1733149302096/Put/seqid=0 2024-12-02T14:21:43,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742375_1551 (size=12207) 2024-12-02T14:21:43,651 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:43,653 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024821679a86974334806faa7e820dcec3_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024821679a86974334806faa7e820dcec3_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:43,654 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/3de31fa980734d6aa3ac0313eb1be0b7, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:43,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/3de31fa980734d6aa3ac0313eb1be0b7 is 175, key is test_row_0/A:col10/1733149303239/Put/seqid=0 2024-12-02T14:21:43,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742376_1552 (size=39549) 2024-12-02T14:21:43,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:43,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:43,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149363899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:43,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:43,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149363899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:43,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:43,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149363908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149363908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:43,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:43,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:43,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:43,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T14:21:44,043 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/b765e9c08d0c481190e3595ff0059a3b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/b765e9c08d0c481190e3595ff0059a3b 2024-12-02T14:21:44,047 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/C of 1a7e8a8b8e1f978fa1c5a7606b30770c into b765e9c08d0c481190e3595ff0059a3b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:44,047 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:44,047 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/C, priority=13, startTime=1733149303208; duration=0sec 2024-12-02T14:21:44,047 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:44,047 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:C 2024-12-02T14:21:44,057 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/3de31fa980734d6aa3ac0313eb1be0b7 2024-12-02T14:21:44,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/b03c7981ff2d42488339bf4d28e2a3ae is 50, key is test_row_0/B:col10/1733149303239/Put/seqid=0 2024-12-02T14:21:44,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742377_1553 (size=12001) 2024-12-02T14:21:44,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:44,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:44,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:44,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:44,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:44,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:44,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:44,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:44,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:44,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:44,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149364419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:44,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/b03c7981ff2d42488339bf4d28e2a3ae 2024-12-02T14:21:44,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/688d81e8125f4ce2ac89e35c770220b1 is 50, key is test_row_0/C:col10/1733149303239/Put/seqid=0 2024-12-02T14:21:44,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742378_1554 (size=12001) 2024-12-02T14:21:44,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:44,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:44,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:44,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:44,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:44,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:44,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:44,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:44,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:44,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:44,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:44,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:44,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:44,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/688d81e8125f4ce2ac89e35c770220b1 2024-12-02T14:21:44,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/3de31fa980734d6aa3ac0313eb1be0b7 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3de31fa980734d6aa3ac0313eb1be0b7 2024-12-02T14:21:44,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3de31fa980734d6aa3ac0313eb1be0b7, entries=200, sequenceid=118, filesize=38.6 K 2024-12-02T14:21:44,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/b03c7981ff2d42488339bf4d28e2a3ae as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/b03c7981ff2d42488339bf4d28e2a3ae 2024-12-02T14:21:44,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/b03c7981ff2d42488339bf4d28e2a3ae, entries=150, sequenceid=118, filesize=11.7 K 2024-12-02T14:21:44,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/688d81e8125f4ce2ac89e35c770220b1 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/688d81e8125f4ce2ac89e35c770220b1 2024-12-02T14:21:44,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/688d81e8125f4ce2ac89e35c770220b1, entries=150, sequenceid=118, filesize=11.7 K 2024-12-02T14:21:44,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 1644ms, sequenceid=118, compaction requested=false 2024-12-02T14:21:44,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:44,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:44,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T14:21:44,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:44,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:44,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:44,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:44,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:44,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:44,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202602a1f83563844b5be32ce84d520697d_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149303282/Put/seqid=0 2024-12-02T14:21:44,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742379_1555 (size=14794) 2024-12-02T14:21:44,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:44,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149364928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:44,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149364932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:44,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:44,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:44,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:44,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149364932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:45,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:45,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:45,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T14:21:45,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:45,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:45,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149365039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:45,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149365039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:45,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:45,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:45,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:45,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149365048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:45,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:45,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:45,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:45,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149365248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:45,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149365249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:45,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149365258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:45,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:45,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:45,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,321 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:45,324 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202602a1f83563844b5be32ce84d520697d_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202602a1f83563844b5be32ce84d520697d_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:45,324 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/d65e40dba4eb4a6e845f65d280c14042, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:45,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/d65e40dba4eb4a6e845f65d280c14042 is 175, key is test_row_0/A:col10/1733149303282/Put/seqid=0 2024-12-02T14:21:45,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742380_1556 (size=39749) 2024-12-02T14:21:45,328 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/d65e40dba4eb4a6e845f65d280c14042 2024-12-02T14:21:45,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/b814e625b10a40f7a97651d9dfe63d1a is 50, key is test_row_0/B:col10/1733149303282/Put/seqid=0 2024-12-02T14:21:45,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742381_1557 (size=12151) 2024-12-02T14:21:45,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/b814e625b10a40f7a97651d9dfe63d1a 2024-12-02T14:21:45,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/7fbeb3bcabcf46e58a7f23cdd3768f61 is 50, key is test_row_0/C:col10/1733149303282/Put/seqid=0 2024-12-02T14:21:45,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742382_1558 (size=12151) 2024-12-02T14:21:45,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/7fbeb3bcabcf46e58a7f23cdd3768f61 2024-12-02T14:21:45,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/d65e40dba4eb4a6e845f65d280c14042 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/d65e40dba4eb4a6e845f65d280c14042 2024-12-02T14:21:45,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/d65e40dba4eb4a6e845f65d280c14042, entries=200, sequenceid=137, filesize=38.8 K 2024-12-02T14:21:45,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/b814e625b10a40f7a97651d9dfe63d1a as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/b814e625b10a40f7a97651d9dfe63d1a 2024-12-02T14:21:45,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/b814e625b10a40f7a97651d9dfe63d1a, entries=150, sequenceid=137, filesize=11.9 K 2024-12-02T14:21:45,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/7fbeb3bcabcf46e58a7f23cdd3768f61 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/7fbeb3bcabcf46e58a7f23cdd3768f61 2024-12-02T14:21:45,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/7fbeb3bcabcf46e58a7f23cdd3768f61, entries=150, sequenceid=137, filesize=11.9 K 2024-12-02T14:21:45,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 447ms, sequenceid=137, compaction requested=true 2024-12-02T14:21:45,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:45,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:21:45,358 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:45,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:45,358 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:45,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:21:45,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:45,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:21:45,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:45,361 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:45,361 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/A is initiating minor compaction (all files) 2024-12-02T14:21:45,362 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/A in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,362 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/cb03941b1fff47419eef69a868b92889, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3de31fa980734d6aa3ac0313eb1be0b7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/d65e40dba4eb4a6e845f65d280c14042] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=107.9 K 2024-12-02T14:21:45,362 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,362 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/cb03941b1fff47419eef69a868b92889, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3de31fa980734d6aa3ac0313eb1be0b7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/d65e40dba4eb4a6e845f65d280c14042] 2024-12-02T14:21:45,362 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:45,362 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/B is initiating minor compaction (all files) 2024-12-02T14:21:45,362 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/B in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,362 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7e1e17b740a5435687479fd683358a25, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/b03c7981ff2d42488339bf4d28e2a3ae, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/b814e625b10a40f7a97651d9dfe63d1a] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=35.5 K 2024-12-02T14:21:45,362 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb03941b1fff47419eef69a868b92889, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733149302096 2024-12-02T14:21:45,362 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e1e17b740a5435687479fd683358a25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733149302096 2024-12-02T14:21:45,363 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3de31fa980734d6aa3ac0313eb1be0b7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733149302756 2024-12-02T14:21:45,363 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting b03c7981ff2d42488339bf4d28e2a3ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733149302756 2024-12-02T14:21:45,363 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting d65e40dba4eb4a6e845f65d280c14042, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733149303270 2024-12-02T14:21:45,363 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting b814e625b10a40f7a97651d9dfe63d1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733149303270 2024-12-02T14:21:45,369 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:45,371 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#B#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:45,371 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/5bec31690b1c4a148ad8aadbcfcf40f8 is 50, key is test_row_0/B:col10/1733149303282/Put/seqid=0 2024-12-02T14:21:45,373 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202b5dcede26cdb42ba9a20ea0b9d8819ca_1a7e8a8b8e1f978fa1c5a7606b30770c store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:45,375 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202b5dcede26cdb42ba9a20ea0b9d8819ca_1a7e8a8b8e1f978fa1c5a7606b30770c, store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:45,375 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b5dcede26cdb42ba9a20ea0b9d8819ca_1a7e8a8b8e1f978fa1c5a7606b30770c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:45,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742383_1559 (size=12459) 2024-12-02T14:21:45,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742384_1560 (size=4469) 2024-12-02T14:21:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:45,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T14:21:45,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:45,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:45,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:45,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:45,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:45,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:45,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024778ec5173ef494985ab3055e749084d_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149304931/Put/seqid=0 2024-12-02T14:21:45,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742385_1561 (size=12304) 2024-12-02T14:21:45,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:45,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:45,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149365489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:45,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:45,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149365558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:45,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149365558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:45,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149365568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:45,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149365598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:45,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:45,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:45,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:45,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:45,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,781 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/5bec31690b1c4a148ad8aadbcfcf40f8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/5bec31690b1c4a148ad8aadbcfcf40f8 2024-12-02T14:21:45,784 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#A#compaction#472 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:45,784 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/93f0aef667cd4c4a8d91c68d6689e5fb is 175, key is test_row_0/A:col10/1733149303282/Put/seqid=0 2024-12-02T14:21:45,786 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/B of 1a7e8a8b8e1f978fa1c5a7606b30770c into 5bec31690b1c4a148ad8aadbcfcf40f8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:45,786 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:45,786 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/B, priority=13, startTime=1733149305358; duration=0sec 2024-12-02T14:21:45,787 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:45,787 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:B 2024-12-02T14:21:45,787 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:45,787 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:45,787 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/C is initiating minor compaction (all files) 2024-12-02T14:21:45,788 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/C in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,788 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/b765e9c08d0c481190e3595ff0059a3b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/688d81e8125f4ce2ac89e35c770220b1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/7fbeb3bcabcf46e58a7f23cdd3768f61] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=35.5 K 2024-12-02T14:21:45,788 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting b765e9c08d0c481190e3595ff0059a3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733149302096 2024-12-02T14:21:45,788 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 688d81e8125f4ce2ac89e35c770220b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733149302756 2024-12-02T14:21:45,788 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fbeb3bcabcf46e58a7f23cdd3768f61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733149303270 2024-12-02T14:21:45,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742386_1562 (size=31413) 2024-12-02T14:21:45,794 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#C#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:45,795 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/5b047de7de004f97895f1d529220261e is 50, key is test_row_0/C:col10/1733149303282/Put/seqid=0 2024-12-02T14:21:45,795 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/93f0aef667cd4c4a8d91c68d6689e5fb as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/93f0aef667cd4c4a8d91c68d6689e5fb 2024-12-02T14:21:45,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742387_1563 (size=12459) 2024-12-02T14:21:45,799 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/A of 1a7e8a8b8e1f978fa1c5a7606b30770c into 93f0aef667cd4c4a8d91c68d6689e5fb(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:45,799 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:45,800 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/A, priority=13, startTime=1733149305358; duration=0sec 2024-12-02T14:21:45,800 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:45,800 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:A 2024-12-02T14:21:45,800 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/5b047de7de004f97895f1d529220261e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/5b047de7de004f97895f1d529220261e 2024-12-02T14:21:45,803 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/C of 1a7e8a8b8e1f978fa1c5a7606b30770c into 5b047de7de004f97895f1d529220261e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:45,803 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:45,803 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/C, priority=13, startTime=1733149305358; duration=0sec 2024-12-02T14:21:45,804 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:45,804 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:C 2024-12-02T14:21:45,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149365808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:45,848 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:45,851 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024778ec5173ef494985ab3055e749084d_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024778ec5173ef494985ab3055e749084d_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:45,852 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/0e15e1fdeb8942eb8ccdc20f0eb91e96, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:45,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/0e15e1fdeb8942eb8ccdc20f0eb91e96 is 175, key is test_row_0/A:col10/1733149304931/Put/seqid=0 2024-12-02T14:21:45,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742388_1564 (size=31105) 2024-12-02T14:21:45,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:45,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:45,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:45,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:46,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149366068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:46,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149366068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:46,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:46,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:46,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149366079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:46,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149366118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:46,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:46,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:46,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:46,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:46,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:46,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:46,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:46,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:46,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:46,255 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/0e15e1fdeb8942eb8ccdc20f0eb91e96 2024-12-02T14:21:46,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/f5feefac6a104ccfaa78471fbda711b7 is 50, key is test_row_0/B:col10/1733149304931/Put/seqid=0 2024-12-02T14:21:46,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742389_1565 (size=12151) 2024-12-02T14:21:46,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:46,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:46,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:46,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:46,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:46,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:46,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:46,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:46,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:46,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:46,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:46,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:46,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149366628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:46,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:46,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/f5feefac6a104ccfaa78471fbda711b7 2024-12-02T14:21:46,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/14b9656bf6eb4ac59c5a0224e5b1fb5d is 50, key is test_row_0/C:col10/1733149304931/Put/seqid=0 2024-12-02T14:21:46,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742390_1566 (size=12151) 2024-12-02T14:21:46,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/14b9656bf6eb4ac59c5a0224e5b1fb5d 2024-12-02T14:21:46,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/0e15e1fdeb8942eb8ccdc20f0eb91e96 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/0e15e1fdeb8942eb8ccdc20f0eb91e96 2024-12-02T14:21:46,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/0e15e1fdeb8942eb8ccdc20f0eb91e96, entries=150, sequenceid=154, filesize=30.4 K 2024-12-02T14:21:46,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/f5feefac6a104ccfaa78471fbda711b7 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f5feefac6a104ccfaa78471fbda711b7 2024-12-02T14:21:46,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f5feefac6a104ccfaa78471fbda711b7, entries=150, sequenceid=154, filesize=11.9 K 2024-12-02T14:21:46,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/14b9656bf6eb4ac59c5a0224e5b1fb5d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/14b9656bf6eb4ac59c5a0224e5b1fb5d 2024-12-02T14:21:46,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/14b9656bf6eb4ac59c5a0224e5b1fb5d, entries=150, sequenceid=154, filesize=11.9 K 2024-12-02T14:21:46,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 1245ms, sequenceid=154, compaction requested=false 2024-12-02T14:21:46,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:46,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-02T14:21:46,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:46,686 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T14:21:46,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:46,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:46,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:46,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:46,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:46,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:46,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029641599ce7254ce28ddaae1a3ffcaa37_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149305488/Put/seqid=0 2024-12-02T14:21:46,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742391_1567 (size=12304) 2024-12-02T14:21:46,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:46,703 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029641599ce7254ce28ddaae1a3ffcaa37_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029641599ce7254ce28ddaae1a3ffcaa37_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:46,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/5769c69e363c4d42beed5c04a5b256b0, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:46,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/5769c69e363c4d42beed5c04a5b256b0 is 175, key is test_row_0/A:col10/1733149305488/Put/seqid=0 2024-12-02T14:21:46,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742392_1568 (size=31105) 2024-12-02T14:21:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:47,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:47,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T14:21:47,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:47,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149367065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:47,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149367078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:47,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149367078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:47,081 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 1a7e8a8b8e1f978fa1c5a7606b30770c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:47,082 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 1a7e8a8b8e1f978fa1c5a7606b30770c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:47,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149367089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:47,090 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 1a7e8a8b8e1f978fa1c5a7606b30770c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:47,107 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/5769c69e363c4d42beed5c04a5b256b0 2024-12-02T14:21:47,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/f764305b01aa45fda90683acf2a45551 is 50, key is test_row_0/B:col10/1733149305488/Put/seqid=0 2024-12-02T14:21:47,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742393_1569 (size=12151) 2024-12-02T14:21:47,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:47,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149367178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:47,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:47,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:47,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:47,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149367388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:47,516 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/f764305b01aa45fda90683acf2a45551 2024-12-02T14:21:47,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/82efd7708fb04ea2b2bef9e2116a43b8 is 50, key is test_row_0/C:col10/1733149305488/Put/seqid=0 2024-12-02T14:21:47,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742394_1570 (size=12151) 2024-12-02T14:21:47,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149367639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:47,641 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_1' in 1a7e8a8b8e1f978fa1c5a7606b30770c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:47,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149367698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:47,926 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/82efd7708fb04ea2b2bef9e2116a43b8 2024-12-02T14:21:47,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/5769c69e363c4d42beed5c04a5b256b0 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/5769c69e363c4d42beed5c04a5b256b0 2024-12-02T14:21:47,931 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/5769c69e363c4d42beed5c04a5b256b0, entries=150, sequenceid=172, filesize=30.4 K 2024-12-02T14:21:47,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/f764305b01aa45fda90683acf2a45551 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f764305b01aa45fda90683acf2a45551 2024-12-02T14:21:47,935 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f764305b01aa45fda90683acf2a45551, entries=150, sequenceid=172, filesize=11.9 K 2024-12-02T14:21:47,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/82efd7708fb04ea2b2bef9e2116a43b8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/82efd7708fb04ea2b2bef9e2116a43b8 2024-12-02T14:21:47,938 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/82efd7708fb04ea2b2bef9e2116a43b8, entries=150, sequenceid=172, filesize=11.9 K 2024-12-02T14:21:47,939 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 1252ms, sequenceid=172, compaction requested=true 2024-12-02T14:21:47,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:47,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:47,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-02T14:21:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-02T14:21:47,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-02T14:21:47,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.0690 sec 2024-12-02T14:21:47,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 5.0720 sec 2024-12-02T14:21:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:48,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T14:21:48,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:48,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:48,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:48,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:48,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:48,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:48,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202321330aa2eb94eba8d51e524ca9e3fac_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149307064/Put/seqid=0 2024-12-02T14:21:48,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742395_1571 (size=12304) 2024-12-02T14:21:48,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149368248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:48,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:48,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:48,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:48,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149368358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:48,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:48,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:48,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:48,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149368569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:48,619 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:48,622 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202321330aa2eb94eba8d51e524ca9e3fac_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202321330aa2eb94eba8d51e524ca9e3fac_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:48,622 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/e064bf70a43c40d4a2e46d24cd4e03cf, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:48,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/e064bf70a43c40d4a2e46d24cd4e03cf is 175, key is test_row_0/A:col10/1733149307064/Put/seqid=0 2024-12-02T14:21:48,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742396_1572 (size=31105) 2024-12-02T14:21:48,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149368878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:48,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:49,027 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/e064bf70a43c40d4a2e46d24cd4e03cf 2024-12-02T14:21:49,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/87e35e5955ec41ec8759b2ed470e14fc is 50, key is test_row_0/B:col10/1733149307064/Put/seqid=0 2024-12-02T14:21:49,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742397_1573 (size=12151) 2024-12-02T14:21:49,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149369089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:49,091 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 1a7e8a8b8e1f978fa1c5a7606b30770c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:49,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149369098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:49,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149369098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:49,101 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 1a7e8a8b8e1f978fa1c5a7606b30770c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:49,101 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 1a7e8a8b8e1f978fa1c5a7606b30770c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:49,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149369388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:49,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/87e35e5955ec41ec8759b2ed470e14fc 2024-12-02T14:21:49,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/48247ddffff34262b916943711271d13 is 50, key is test_row_0/C:col10/1733149307064/Put/seqid=0 2024-12-02T14:21:49,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742398_1574 (size=12151) 2024-12-02T14:21:49,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/48247ddffff34262b916943711271d13 2024-12-02T14:21:49,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/e064bf70a43c40d4a2e46d24cd4e03cf as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/e064bf70a43c40d4a2e46d24cd4e03cf 2024-12-02T14:21:49,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/e064bf70a43c40d4a2e46d24cd4e03cf, entries=150, sequenceid=190, filesize=30.4 K 2024-12-02T14:21:49,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/87e35e5955ec41ec8759b2ed470e14fc as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/87e35e5955ec41ec8759b2ed470e14fc 2024-12-02T14:21:49,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/87e35e5955ec41ec8759b2ed470e14fc, entries=150, sequenceid=190, filesize=11.9 K 2024-12-02T14:21:49,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/48247ddffff34262b916943711271d13 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/48247ddffff34262b916943711271d13 2024-12-02T14:21:49,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/48247ddffff34262b916943711271d13, entries=150, sequenceid=190, filesize=11.9 K 2024-12-02T14:21:49,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 1246ms, sequenceid=190, compaction requested=true 2024-12-02T14:21:49,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:49,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:21:49,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:49,457 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:21:49,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:21:49,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:49,457 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:21:49,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:21:49,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:49,458 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:21:49,458 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:21:49,458 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/B is initiating minor compaction (all files) 2024-12-02T14:21:49,458 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/A is initiating minor compaction (all files) 2024-12-02T14:21:49,458 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/B in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:49,458 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/A in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:49,458 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/5bec31690b1c4a148ad8aadbcfcf40f8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f5feefac6a104ccfaa78471fbda711b7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f764305b01aa45fda90683acf2a45551, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/87e35e5955ec41ec8759b2ed470e14fc] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=47.8 K 2024-12-02T14:21:49,458 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/93f0aef667cd4c4a8d91c68d6689e5fb, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/0e15e1fdeb8942eb8ccdc20f0eb91e96, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/5769c69e363c4d42beed5c04a5b256b0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/e064bf70a43c40d4a2e46d24cd4e03cf] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=121.8 K 2024-12-02T14:21:49,458 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:49,458 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/93f0aef667cd4c4a8d91c68d6689e5fb, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/0e15e1fdeb8942eb8ccdc20f0eb91e96, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/5769c69e363c4d42beed5c04a5b256b0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/e064bf70a43c40d4a2e46d24cd4e03cf] 2024-12-02T14:21:49,459 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bec31690b1c4a148ad8aadbcfcf40f8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733149303270 2024-12-02T14:21:49,459 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93f0aef667cd4c4a8d91c68d6689e5fb, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733149303270 2024-12-02T14:21:49,459 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting f5feefac6a104ccfaa78471fbda711b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733149304930 2024-12-02T14:21:49,459 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e15e1fdeb8942eb8ccdc20f0eb91e96, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733149304930 2024-12-02T14:21:49,459 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting f764305b01aa45fda90683acf2a45551, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733149305472 2024-12-02T14:21:49,459 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5769c69e363c4d42beed5c04a5b256b0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733149305472 2024-12-02T14:21:49,459 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 87e35e5955ec41ec8759b2ed470e14fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733149307056 2024-12-02T14:21:49,459 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting e064bf70a43c40d4a2e46d24cd4e03cf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733149307056 2024-12-02T14:21:49,469 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:49,470 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#B#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:49,470 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120293564bdfca654cfc87fa024f1a08783a_1a7e8a8b8e1f978fa1c5a7606b30770c store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:49,471 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/2eb3149f62624731bc729bc241ed5b58 is 50, key is test_row_0/B:col10/1733149307064/Put/seqid=0 2024-12-02T14:21:49,472 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120293564bdfca654cfc87fa024f1a08783a_1a7e8a8b8e1f978fa1c5a7606b30770c, store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:49,473 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120293564bdfca654cfc87fa024f1a08783a_1a7e8a8b8e1f978fa1c5a7606b30770c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:49,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742400_1576 (size=4469) 2024-12-02T14:21:49,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742399_1575 (size=12595) 2024-12-02T14:21:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:49,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T14:21:49,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:49,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:49,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:49,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:49,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:49,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:49,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120239095b622f5640d2a475eded42e6efb0_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149308228/Put/seqid=0 2024-12-02T14:21:49,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742401_1577 (size=14794) 2024-12-02T14:21:49,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149369721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:49,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149369828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:49,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:49,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:49,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:49,876 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#A#compaction#484 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:49,877 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/3f8937e8508543e4a9827cb8ae191dbd is 175, key is test_row_0/A:col10/1733149307064/Put/seqid=0 2024-12-02T14:21:49,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742402_1578 (size=31549) 2024-12-02T14:21:49,880 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/2eb3149f62624731bc729bc241ed5b58 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/2eb3149f62624731bc729bc241ed5b58 2024-12-02T14:21:49,884 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/B of 1a7e8a8b8e1f978fa1c5a7606b30770c into 2eb3149f62624731bc729bc241ed5b58(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:49,884 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:49,884 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/B, priority=12, startTime=1733149309457; duration=0sec 2024-12-02T14:21:49,884 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:49,884 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:B 2024-12-02T14:21:49,884 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:21:49,885 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:21:49,885 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/C is initiating minor compaction (all files) 2024-12-02T14:21:49,885 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/C in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:49,885 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/5b047de7de004f97895f1d529220261e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/14b9656bf6eb4ac59c5a0224e5b1fb5d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/82efd7708fb04ea2b2bef9e2116a43b8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/48247ddffff34262b916943711271d13] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=47.8 K 2024-12-02T14:21:49,885 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b047de7de004f97895f1d529220261e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733149303270 2024-12-02T14:21:49,886 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 14b9656bf6eb4ac59c5a0224e5b1fb5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733149304930 2024-12-02T14:21:49,886 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 82efd7708fb04ea2b2bef9e2116a43b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733149305472 2024-12-02T14:21:49,886 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 48247ddffff34262b916943711271d13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733149307056 2024-12-02T14:21:49,892 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#C#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:49,893 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/49b91def1549447b9f45cd083dd1171d is 50, key is test_row_0/C:col10/1733149307064/Put/seqid=0 2024-12-02T14:21:49,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742403_1579 (size=12595) 2024-12-02T14:21:50,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149370039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:50,062 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:50,064 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120239095b622f5640d2a475eded42e6efb0_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120239095b622f5640d2a475eded42e6efb0_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:50,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/c23affedc9994c76a9ec06ad90d4fd80, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:50,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/c23affedc9994c76a9ec06ad90d4fd80 is 175, key is test_row_0/A:col10/1733149308228/Put/seqid=0 2024-12-02T14:21:50,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742404_1580 (size=39749) 2024-12-02T14:21:50,283 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/3f8937e8508543e4a9827cb8ae191dbd as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3f8937e8508543e4a9827cb8ae191dbd 2024-12-02T14:21:50,286 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/A of 1a7e8a8b8e1f978fa1c5a7606b30770c into 3f8937e8508543e4a9827cb8ae191dbd(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:50,286 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:50,286 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/A, priority=12, startTime=1733149309457; duration=0sec 2024-12-02T14:21:50,286 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:50,286 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:A 2024-12-02T14:21:50,299 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/49b91def1549447b9f45cd083dd1171d as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/49b91def1549447b9f45cd083dd1171d 2024-12-02T14:21:50,303 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/C of 1a7e8a8b8e1f978fa1c5a7606b30770c into 49b91def1549447b9f45cd083dd1171d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:50,303 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:50,303 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/C, priority=12, startTime=1733149309457; duration=0sec 2024-12-02T14:21:50,303 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:50,303 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:C 2024-12-02T14:21:50,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:50,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149370348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:50,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:50,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:50,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:50,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:50,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149370398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:50,402 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_1' in 1a7e8a8b8e1f978fa1c5a7606b30770c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:50,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:50,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:50,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:50,469 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/c23affedc9994c76a9ec06ad90d4fd80 2024-12-02T14:21:50,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/9af8f0e30d564021bcf4d78268ea49b5 is 50, key is test_row_0/B:col10/1733149308228/Put/seqid=0 2024-12-02T14:21:50,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742405_1581 (size=12151) 2024-12-02T14:21:50,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149370859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:50,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/9af8f0e30d564021bcf4d78268ea49b5 2024-12-02T14:21:50,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/0b45d6c0bd304f9d8642a9b53a267576 is 50, key is test_row_0/C:col10/1733149308228/Put/seqid=0 2024-12-02T14:21:50,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742406_1582 (size=12151) 2024-12-02T14:21:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-02T14:21:51,059 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:51,059 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:21:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-02T14:21:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T14:21:51,060 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:21:51,061 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:21:51,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:21:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T14:21:51,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T14:21:51,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:51,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:51,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:51,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:51,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:51,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/0b45d6c0bd304f9d8642a9b53a267576 2024-12-02T14:21:51,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/c23affedc9994c76a9ec06ad90d4fd80 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/c23affedc9994c76a9ec06ad90d4fd80 2024-12-02T14:21:51,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/c23affedc9994c76a9ec06ad90d4fd80, entries=200, sequenceid=205, filesize=38.8 K 2024-12-02T14:21:51,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/9af8f0e30d564021bcf4d78268ea49b5 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/9af8f0e30d564021bcf4d78268ea49b5 2024-12-02T14:21:51,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/9af8f0e30d564021bcf4d78268ea49b5, entries=150, sequenceid=205, filesize=11.9 K 2024-12-02T14:21:51,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/0b45d6c0bd304f9d8642a9b53a267576 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/0b45d6c0bd304f9d8642a9b53a267576 2024-12-02T14:21:51,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/0b45d6c0bd304f9d8642a9b53a267576, entries=150, sequenceid=205, filesize=11.9 K 2024-12-02T14:21:51,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 1646ms, sequenceid=205, compaction requested=false 2024-12-02T14:21:51,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:51,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T14:21:51,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:51,367 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T14:21:51,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:51,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:51,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:51,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:51,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:51,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:51,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202dbdb0f8bfaf749fca47baf334d2f13d8_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149309707/Put/seqid=0 2024-12-02T14:21:51,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742407_1583 (size=12304) 2024-12-02T14:21:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T14:21:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T14:21:51,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:51,779 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202dbdb0f8bfaf749fca47baf334d2f13d8_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dbdb0f8bfaf749fca47baf334d2f13d8_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:51,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/8518b7aef05540488f43c1e65ca01ed7, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:51,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/8518b7aef05540488f43c1e65ca01ed7 is 175, key is test_row_0/A:col10/1733149309707/Put/seqid=0 2024-12-02T14:21:51,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742408_1584 (size=31105) 2024-12-02T14:21:51,784 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/8518b7aef05540488f43c1e65ca01ed7 2024-12-02T14:21:51,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/2e48869eac5a46fa8143fcbbb4b050e8 is 50, key is test_row_0/B:col10/1733149309707/Put/seqid=0 2024-12-02T14:21:51,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742409_1585 (size=12151) 2024-12-02T14:21:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:51,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:51,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149371910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:51,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:52,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149372018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:52,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:52,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:52,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:52,193 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/2e48869eac5a46fa8143fcbbb4b050e8 2024-12-02T14:21:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T14:21:52,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/711d30f4631a4002adcdec65c291cc6b is 50, key is test_row_0/C:col10/1733149309707/Put/seqid=0 2024-12-02T14:21:52,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742410_1586 (size=12151) 2024-12-02T14:21:52,219 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/711d30f4631a4002adcdec65c291cc6b 2024-12-02T14:21:52,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/8518b7aef05540488f43c1e65ca01ed7 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8518b7aef05540488f43c1e65ca01ed7 2024-12-02T14:21:52,226 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8518b7aef05540488f43c1e65ca01ed7, entries=150, sequenceid=225, filesize=30.4 K 2024-12-02T14:21:52,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/2e48869eac5a46fa8143fcbbb4b050e8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/2e48869eac5a46fa8143fcbbb4b050e8 2024-12-02T14:21:52,229 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/2e48869eac5a46fa8143fcbbb4b050e8, entries=150, sequenceid=225, filesize=11.9 K 2024-12-02T14:21:52,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/711d30f4631a4002adcdec65c291cc6b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/711d30f4631a4002adcdec65c291cc6b 2024-12-02T14:21:52,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149372228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:52,234 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/711d30f4631a4002adcdec65c291cc6b, entries=150, sequenceid=225, filesize=11.9 K 2024-12-02T14:21:52,234 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 867ms, sequenceid=225, compaction requested=true 2024-12-02T14:21:52,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:52,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:52,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-02T14:21:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-02T14:21:52,237 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-02T14:21:52,237 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1750 sec 2024-12-02T14:21:52,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.1790 sec 2024-12-02T14:21:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:52,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T14:21:52,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:52,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:52,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:52,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:52,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:52,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:52,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026815bee4dbfd444b9466f84bb182eb85_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149311881/Put/seqid=0 2024-12-02T14:21:52,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742411_1587 (size=14794) 2024-12-02T14:21:52,440 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:52,443 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026815bee4dbfd444b9466f84bb182eb85_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026815bee4dbfd444b9466f84bb182eb85_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:52,443 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/3581319b2db04ae68ab7163ec63be7c2, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:52,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/3581319b2db04ae68ab7163ec63be7c2 is 175, key is test_row_0/A:col10/1733149311881/Put/seqid=0 2024-12-02T14:21:52,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742412_1588 (size=39749) 2024-12-02T14:21:52,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149372481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:52,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149372539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:52,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:52,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149372588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:52,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149372798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:52,847 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/3581319b2db04ae68ab7163ec63be7c2 2024-12-02T14:21:52,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/7be1eeeeaf1a4e389a677623d129db82 is 50, key is test_row_0/B:col10/1733149311881/Put/seqid=0 2024-12-02T14:21:52,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742413_1589 (size=12151) 2024-12-02T14:21:53,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149373049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:53,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149373108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:53,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:53,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:53,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:53,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52926 deadline: 1733149373118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:53,121 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 1a7e8a8b8e1f978fa1c5a7606b30770c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:53,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:53,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:53,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:53,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52894 deadline: 1733149373129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:53,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52922 deadline: 1733149373129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:53,130 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_0' in 1a7e8a8b8e1f978fa1c5a7606b30770c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:53,130 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 1a7e8a8b8e1f978fa1c5a7606b30770c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T14:21:53,209 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:53,209 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:21:53,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-02T14:21:53,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T14:21:53,211 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:21:53,211 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:21:53,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:21:53,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/7be1eeeeaf1a4e389a677623d129db82 2024-12-02T14:21:53,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/464780a8a4dc4d9e9cacf7ec23f0dbce is 50, key is test_row_0/C:col10/1733149311881/Put/seqid=0 2024-12-02T14:21:53,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742414_1590 (size=12151) 2024-12-02T14:21:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T14:21:53,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-02T14:21:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:53,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:53,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:53,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:53,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-02T14:21:53,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:53,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:53,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:53,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:53,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:53,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T14:21:53,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149373619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:53,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/464780a8a4dc4d9e9cacf7ec23f0dbce 2024-12-02T14:21:53,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-02T14:21:53,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:53,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:53,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:53,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/3581319b2db04ae68ab7163ec63be7c2 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3581319b2db04ae68ab7163ec63be7c2 2024-12-02T14:21:53,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:53,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:53,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3581319b2db04ae68ab7163ec63be7c2, entries=200, sequenceid=241, filesize=38.8 K 2024-12-02T14:21:53,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/7be1eeeeaf1a4e389a677623d129db82 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7be1eeeeaf1a4e389a677623d129db82 2024-12-02T14:21:53,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7be1eeeeaf1a4e389a677623d129db82, entries=150, sequenceid=241, filesize=11.9 K 2024-12-02T14:21:53,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/464780a8a4dc4d9e9cacf7ec23f0dbce as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/464780a8a4dc4d9e9cacf7ec23f0dbce 2024-12-02T14:21:53,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/464780a8a4dc4d9e9cacf7ec23f0dbce, entries=150, sequenceid=241, filesize=11.9 K 2024-12-02T14:21:53,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 1246ms, sequenceid=241, compaction requested=true 2024-12-02T14:21:53,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:53,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:21:53,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:53,678 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:21:53,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:21:53,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:53,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:21:53,678 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:21:53,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:53,678 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142152 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:21:53,678 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:21:53,679 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/A is initiating minor compaction (all files) 2024-12-02T14:21:53,679 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/B is initiating minor compaction (all files) 2024-12-02T14:21:53,679 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/A in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:53,679 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/B in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:53,679 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/2eb3149f62624731bc729bc241ed5b58, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/9af8f0e30d564021bcf4d78268ea49b5, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/2e48869eac5a46fa8143fcbbb4b050e8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7be1eeeeaf1a4e389a677623d129db82] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=47.9 K 2024-12-02T14:21:53,679 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3f8937e8508543e4a9827cb8ae191dbd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/c23affedc9994c76a9ec06ad90d4fd80, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8518b7aef05540488f43c1e65ca01ed7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3581319b2db04ae68ab7163ec63be7c2] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=138.8 K 2024-12-02T14:21:53,679 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:53,679 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3f8937e8508543e4a9827cb8ae191dbd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/c23affedc9994c76a9ec06ad90d4fd80, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8518b7aef05540488f43c1e65ca01ed7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3581319b2db04ae68ab7163ec63be7c2] 2024-12-02T14:21:53,679 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eb3149f62624731bc729bc241ed5b58, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733149307056 2024-12-02T14:21:53,679 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f8937e8508543e4a9827cb8ae191dbd, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733149307056 2024-12-02T14:21:53,679 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting c23affedc9994c76a9ec06ad90d4fd80, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733149308228 2024-12-02T14:21:53,679 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 9af8f0e30d564021bcf4d78268ea49b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733149308228 2024-12-02T14:21:53,680 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8518b7aef05540488f43c1e65ca01ed7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733149309670 2024-12-02T14:21:53,680 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e48869eac5a46fa8143fcbbb4b050e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733149309670 2024-12-02T14:21:53,680 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3581319b2db04ae68ab7163ec63be7c2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733149311881 2024-12-02T14:21:53,680 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 7be1eeeeaf1a4e389a677623d129db82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733149311881 2024-12-02T14:21:53,686 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#B#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:53,687 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/38cf5f1bec554d44b8d5fde3161fa3c0 is 50, key is test_row_0/B:col10/1733149311881/Put/seqid=0 2024-12-02T14:21:53,689 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:53,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742415_1591 (size=12731) 2024-12-02T14:21:53,690 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412027cdb5074bb3143a1966760d782bb71c7_1a7e8a8b8e1f978fa1c5a7606b30770c store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:53,693 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412027cdb5074bb3143a1966760d782bb71c7_1a7e8a8b8e1f978fa1c5a7606b30770c, store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:53,693 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027cdb5074bb3143a1966760d782bb71c7_1a7e8a8b8e1f978fa1c5a7606b30770c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:53,697 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/38cf5f1bec554d44b8d5fde3161fa3c0 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/38cf5f1bec554d44b8d5fde3161fa3c0 2024-12-02T14:21:53,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742416_1592 (size=4469) 2024-12-02T14:21:53,699 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#A#compaction#496 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:53,699 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/f46a62cde4b74affb003329735557012 is 175, key is test_row_0/A:col10/1733149311881/Put/seqid=0 2024-12-02T14:21:53,702 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/B of 1a7e8a8b8e1f978fa1c5a7606b30770c into 38cf5f1bec554d44b8d5fde3161fa3c0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:53,702 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:53,702 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/B, priority=12, startTime=1733149313678; duration=0sec 2024-12-02T14:21:53,702 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:53,702 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:B 2024-12-02T14:21:53,703 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T14:21:53,705 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T14:21:53,705 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/C is initiating minor compaction (all files) 2024-12-02T14:21:53,705 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/C in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:53,706 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/49b91def1549447b9f45cd083dd1171d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/0b45d6c0bd304f9d8642a9b53a267576, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/711d30f4631a4002adcdec65c291cc6b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/464780a8a4dc4d9e9cacf7ec23f0dbce] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=47.9 K 2024-12-02T14:21:53,706 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 49b91def1549447b9f45cd083dd1171d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733149307056 2024-12-02T14:21:53,706 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b45d6c0bd304f9d8642a9b53a267576, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733149308228 2024-12-02T14:21:53,706 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 711d30f4631a4002adcdec65c291cc6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733149309670 2024-12-02T14:21:53,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742417_1593 (size=31685) 2024-12-02T14:21:53,707 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 464780a8a4dc4d9e9cacf7ec23f0dbce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733149311881 2024-12-02T14:21:53,713 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#C#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:53,713 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/c638c9a79d8745be99b0bbe072e199d1 is 50, key is test_row_0/C:col10/1733149311881/Put/seqid=0 2024-12-02T14:21:53,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742418_1594 (size=12731) 2024-12-02T14:21:53,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-02T14:21:53,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:53,821 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T14:21:53,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:53,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:53,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:53,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:53,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:53,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:53,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d33962a2e15d460a80b6cb09e6bea135_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149312480/Put/seqid=0 2024-12-02T14:21:53,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742419_1595 (size=12304) 2024-12-02T14:21:53,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:53,831 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d33962a2e15d460a80b6cb09e6bea135_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d33962a2e15d460a80b6cb09e6bea135_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:53,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/e8d42fcabf3e40488516cf877e3e4621, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:53,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/e8d42fcabf3e40488516cf877e3e4621 is 175, key is test_row_0/A:col10/1733149312480/Put/seqid=0 2024-12-02T14:21:53,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742420_1596 (size=31105) 2024-12-02T14:21:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T14:21:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:54,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:54,111 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/f46a62cde4b74affb003329735557012 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/f46a62cde4b74affb003329735557012 2024-12-02T14:21:54,115 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/A of 1a7e8a8b8e1f978fa1c5a7606b30770c into f46a62cde4b74affb003329735557012(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:54,115 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:54,115 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/A, priority=12, startTime=1733149313678; duration=0sec 2024-12-02T14:21:54,115 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:54,115 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:A 2024-12-02T14:21:54,120 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/c638c9a79d8745be99b0bbe072e199d1 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/c638c9a79d8745be99b0bbe072e199d1 2024-12-02T14:21:54,124 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/C of 1a7e8a8b8e1f978fa1c5a7606b30770c into c638c9a79d8745be99b0bbe072e199d1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:54,124 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:54,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:54,124 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/C, priority=12, startTime=1733149313678; duration=0sec 2024-12-02T14:21:54,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149374121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:54,124 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:54,124 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:C 2024-12-02T14:21:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:54,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149374228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:54,236 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/e8d42fcabf3e40488516cf877e3e4621 2024-12-02T14:21:54,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/9f3b23a99a7545418565c05c7443542f is 50, key is test_row_0/B:col10/1733149312480/Put/seqid=0 2024-12-02T14:21:54,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742421_1597 (size=12151) 2024-12-02T14:21:54,245 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/9f3b23a99a7545418565c05c7443542f 2024-12-02T14:21:54,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/33919bfd70e84ca5bd8d6fa29b108566 is 50, key is test_row_0/C:col10/1733149312480/Put/seqid=0 2024-12-02T14:21:54,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742422_1598 (size=12151) 2024-12-02T14:21:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T14:21:54,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149374438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:54,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:54,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:54,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:54,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149374628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:54,630 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_1' in 1a7e8a8b8e1f978fa1c5a7606b30770c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:54,653 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/33919bfd70e84ca5bd8d6fa29b108566 2024-12-02T14:21:54,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/e8d42fcabf3e40488516cf877e3e4621 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/e8d42fcabf3e40488516cf877e3e4621 2024-12-02T14:21:54,659 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/e8d42fcabf3e40488516cf877e3e4621, entries=150, sequenceid=258, filesize=30.4 K 2024-12-02T14:21:54,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/9f3b23a99a7545418565c05c7443542f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/9f3b23a99a7545418565c05c7443542f 2024-12-02T14:21:54,662 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/9f3b23a99a7545418565c05c7443542f, entries=150, sequenceid=258, filesize=11.9 K 2024-12-02T14:21:54,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/33919bfd70e84ca5bd8d6fa29b108566 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/33919bfd70e84ca5bd8d6fa29b108566 2024-12-02T14:21:54,665 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/33919bfd70e84ca5bd8d6fa29b108566, entries=150, sequenceid=258, filesize=11.9 K 2024-12-02T14:21:54,666 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 845ms, sequenceid=258, compaction requested=false 2024-12-02T14:21:54,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:54,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:54,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-02T14:21:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-02T14:21:54,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-02T14:21:54,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4560 sec 2024-12-02T14:21:54,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4600 sec 2024-12-02T14:21:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:54,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T14:21:54,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:54,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:54,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:54,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:54,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:54,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:54,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027a5103bddca647a485b6c327d945c3c8_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149314088/Put/seqid=0 2024-12-02T14:21:54,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742423_1599 (size=12454) 2024-12-02T14:21:54,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149374794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:54,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:54,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:54,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:54,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149374909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:55,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149375118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:55,160 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:55,162 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027a5103bddca647a485b6c327d945c3c8_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027a5103bddca647a485b6c327d945c3c8_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:55,163 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/f92b9f69e60840d7b2ef77e1fe67d221, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:55,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/f92b9f69e60840d7b2ef77e1fe67d221 is 175, key is test_row_0/A:col10/1733149314088/Put/seqid=0 2024-12-02T14:21:55,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742424_1600 (size=31255) 2024-12-02T14:21:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T14:21:55,358 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:55,359 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:21:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-02T14:21:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T14:21:55,360 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:21:55,361 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:21:55,361 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:21:55,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149375429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:55,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:55,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:55,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T14:21:55,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-02T14:21:55,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:55,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:55,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:55,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:55,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:55,576 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/f92b9f69e60840d7b2ef77e1fe67d221 2024-12-02T14:21:55,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/7070bff9ccc046d6a78aed6eb355deee is 50, key is test_row_0/B:col10/1733149314088/Put/seqid=0 2024-12-02T14:21:55,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742425_1601 (size=12301) 2024-12-02T14:21:55,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-02T14:21:55,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:55,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:55,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:55,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:55,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T14:21:55,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-02T14:21:55,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:55,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:55,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:55,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:55,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:55,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149375939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:55,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-02T14:21:55,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:55,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:55,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:55,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:55,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T14:21:56,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/7070bff9ccc046d6a78aed6eb355deee 2024-12-02T14:21:56,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/948ebafd448d4f6a9270589ff3dc6e8c is 50, key is test_row_0/C:col10/1733149314088/Put/seqid=0 2024-12-02T14:21:56,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742426_1602 (size=12301) 2024-12-02T14:21:56,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-02T14:21:56,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:56,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:56,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:56,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:56,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:56,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-02T14:21:56,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:56,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:56,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:56,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:56,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:56,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/948ebafd448d4f6a9270589ff3dc6e8c 2024-12-02T14:21:56,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/f92b9f69e60840d7b2ef77e1fe67d221 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/f92b9f69e60840d7b2ef77e1fe67d221 2024-12-02T14:21:56,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/f92b9f69e60840d7b2ef77e1fe67d221, entries=150, sequenceid=277, filesize=30.5 K 2024-12-02T14:21:56,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/7070bff9ccc046d6a78aed6eb355deee as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7070bff9ccc046d6a78aed6eb355deee 2024-12-02T14:21:56,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7070bff9ccc046d6a78aed6eb355deee, entries=150, sequenceid=277, filesize=12.0 K 2024-12-02T14:21:56,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/948ebafd448d4f6a9270589ff3dc6e8c as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/948ebafd448d4f6a9270589ff3dc6e8c 2024-12-02T14:21:56,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/948ebafd448d4f6a9270589ff3dc6e8c, entries=150, sequenceid=277, filesize=12.0 K 2024-12-02T14:21:56,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 1673ms, sequenceid=277, compaction requested=true 2024-12-02T14:21:56,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:56,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:21:56,423 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:56,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:56,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:21:56,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:56,423 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:56,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:21:56,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:56,424 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:56,424 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94045 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:56,424 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/A is initiating minor compaction (all files) 2024-12-02T14:21:56,424 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/B is initiating minor compaction (all files) 2024-12-02T14:21:56,424 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/B in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:56,424 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/A in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:56,424 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/38cf5f1bec554d44b8d5fde3161fa3c0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/9f3b23a99a7545418565c05c7443542f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7070bff9ccc046d6a78aed6eb355deee] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=36.3 K 2024-12-02T14:21:56,424 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/f46a62cde4b74affb003329735557012, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/e8d42fcabf3e40488516cf877e3e4621, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/f92b9f69e60840d7b2ef77e1fe67d221] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=91.8 K 2024-12-02T14:21:56,424 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:56,424 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/f46a62cde4b74affb003329735557012, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/e8d42fcabf3e40488516cf877e3e4621, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/f92b9f69e60840d7b2ef77e1fe67d221] 2024-12-02T14:21:56,425 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 38cf5f1bec554d44b8d5fde3161fa3c0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733149311881 2024-12-02T14:21:56,425 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting f46a62cde4b74affb003329735557012, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733149311881 2024-12-02T14:21:56,425 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3b23a99a7545418565c05c7443542f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733149312461 2024-12-02T14:21:56,425 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8d42fcabf3e40488516cf877e3e4621, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733149312461 2024-12-02T14:21:56,425 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 7070bff9ccc046d6a78aed6eb355deee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733149314088 2024-12-02T14:21:56,425 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting f92b9f69e60840d7b2ef77e1fe67d221, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733149314088 2024-12-02T14:21:56,431 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:56,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-02T14:21:56,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:56,432 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T14:21:56,432 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#B#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:56,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:56,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:56,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:56,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:56,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:56,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:56,433 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/f8163baf63684781be6a914ca019985f is 50, key is test_row_0/B:col10/1733149314088/Put/seqid=0 2024-12-02T14:21:56,436 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120216f4c27273034b79b8e8b18ee4b2592d_1a7e8a8b8e1f978fa1c5a7606b30770c store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:56,438 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120216f4c27273034b79b8e8b18ee4b2592d_1a7e8a8b8e1f978fa1c5a7606b30770c, store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:56,438 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120216f4c27273034b79b8e8b18ee4b2592d_1a7e8a8b8e1f978fa1c5a7606b30770c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:56,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742427_1603 (size=12983) 2024-12-02T14:21:56,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b71d4229e8794fd7b98d0b9d3c505377_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149314789/Put/seqid=0 2024-12-02T14:21:56,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742428_1604 (size=4469) 2024-12-02T14:21:56,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742429_1605 (size=12454) 2024-12-02T14:21:56,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:56,450 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b71d4229e8794fd7b98d0b9d3c505377_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b71d4229e8794fd7b98d0b9d3c505377_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:56,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/b8838c8456d14cd589097bbd06d60a9e, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:56,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/b8838c8456d14cd589097bbd06d60a9e is 175, key is test_row_0/A:col10/1733149314789/Put/seqid=0 2024-12-02T14:21:56,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742430_1606 (size=31255) 2024-12-02T14:21:56,454 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/b8838c8456d14cd589097bbd06d60a9e 2024-12-02T14:21:56,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/0874de10c50d44e4be0161db47f6f37b is 50, key is test_row_0/B:col10/1733149314789/Put/seqid=0 2024-12-02T14:21:56,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742431_1607 (size=12301) 2024-12-02T14:21:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T14:21:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:56,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:56,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149376702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:56,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:56,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:56,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:56,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149376808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:56,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:56,845 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#A#compaction#505 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:56,846 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/f8163baf63684781be6a914ca019985f as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f8163baf63684781be6a914ca019985f 2024-12-02T14:21:56,846 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/8c94e6672b514dddacfb1b88229508c4 is 175, key is test_row_0/A:col10/1733149314088/Put/seqid=0 2024-12-02T14:21:56,852 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/B of 1a7e8a8b8e1f978fa1c5a7606b30770c into f8163baf63684781be6a914ca019985f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:56,852 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:56,852 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/B, priority=13, startTime=1733149316423; duration=0sec 2024-12-02T14:21:56,852 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:56,853 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:B 2024-12-02T14:21:56,853 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:56,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742432_1608 (size=31937) 2024-12-02T14:21:56,853 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:56,854 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/C is initiating minor compaction (all files) 2024-12-02T14:21:56,854 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/C in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:56,854 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/c638c9a79d8745be99b0bbe072e199d1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/33919bfd70e84ca5bd8d6fa29b108566, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/948ebafd448d4f6a9270589ff3dc6e8c] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=36.3 K 2024-12-02T14:21:56,854 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting c638c9a79d8745be99b0bbe072e199d1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733149311881 2024-12-02T14:21:56,854 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 33919bfd70e84ca5bd8d6fa29b108566, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733149312461 2024-12-02T14:21:56,854 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 948ebafd448d4f6a9270589ff3dc6e8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733149314088 2024-12-02T14:21:56,860 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#C#compaction#508 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:56,860 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/791d79d3d1df45b28ad6d95f970806cd is 50, key is test_row_0/C:col10/1733149314088/Put/seqid=0 2024-12-02T14:21:56,862 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/0874de10c50d44e4be0161db47f6f37b 2024-12-02T14:21:56,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742433_1609 (size=12983) 2024-12-02T14:21:56,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/4b0c54ccae8147679ccbb0a7ef1309ac is 50, key is test_row_0/C:col10/1733149314789/Put/seqid=0 2024-12-02T14:21:56,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742434_1610 (size=12301) 2024-12-02T14:21:56,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149376959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:56,961 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 1a7e8a8b8e1f978fa1c5a7606b30770c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:56,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:57,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149377018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:57,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:57,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:57,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:57,048 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T14:21:57,257 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/8c94e6672b514dddacfb1b88229508c4 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8c94e6672b514dddacfb1b88229508c4 2024-12-02T14:21:57,260 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/A of 1a7e8a8b8e1f978fa1c5a7606b30770c into 8c94e6672b514dddacfb1b88229508c4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:57,260 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:57,260 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/A, priority=13, startTime=1733149316423; duration=0sec 2024-12-02T14:21:57,260 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:57,260 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:A 2024-12-02T14:21:57,267 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/791d79d3d1df45b28ad6d95f970806cd as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/791d79d3d1df45b28ad6d95f970806cd 2024-12-02T14:21:57,270 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/C of 1a7e8a8b8e1f978fa1c5a7606b30770c into 791d79d3d1df45b28ad6d95f970806cd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:57,270 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:57,270 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/C, priority=13, startTime=1733149316423; duration=0sec 2024-12-02T14:21:57,270 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:57,270 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:C 2024-12-02T14:21:57,271 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/4b0c54ccae8147679ccbb0a7ef1309ac 2024-12-02T14:21:57,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/b8838c8456d14cd589097bbd06d60a9e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/b8838c8456d14cd589097bbd06d60a9e 2024-12-02T14:21:57,277 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/b8838c8456d14cd589097bbd06d60a9e, entries=150, sequenceid=292, filesize=30.5 K 2024-12-02T14:21:57,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/0874de10c50d44e4be0161db47f6f37b as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/0874de10c50d44e4be0161db47f6f37b 2024-12-02T14:21:57,280 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/0874de10c50d44e4be0161db47f6f37b, entries=150, sequenceid=292, filesize=12.0 K 2024-12-02T14:21:57,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/4b0c54ccae8147679ccbb0a7ef1309ac as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/4b0c54ccae8147679ccbb0a7ef1309ac 2024-12-02T14:21:57,283 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/4b0c54ccae8147679ccbb0a7ef1309ac, entries=150, sequenceid=292, filesize=12.0 K 2024-12-02T14:21:57,284 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 852ms, sequenceid=292, compaction requested=false 2024-12-02T14:21:57,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:57,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:57,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-02T14:21:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-02T14:21:57,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-02T14:21:57,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9250 sec 2024-12-02T14:21:57,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.9280 sec 2024-12-02T14:21:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:57,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T14:21:57,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:57,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:57,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:57,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:57,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:57,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:57,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202dc18c03745614de9bfc91088a8d85c83_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149317328/Put/seqid=0 2024-12-02T14:21:57,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742435_1611 (size=12454) 2024-12-02T14:21:57,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149377373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:57,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:57,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:57,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:57,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149377488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T14:21:57,509 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:21:57,509 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T14:21:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-02T14:21:57,511 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T14:21:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T14:21:57,511 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T14:21:57,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T14:21:57,571 INFO [Thread-2348 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2348. 2024-12-02T14:21:57,571 DEBUG [Thread-2348 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:21:57,571 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:57,571 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:57,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:57,571 INFO [Thread-2352 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2352. 2024-12-02T14:21:57,571 DEBUG [Thread-2352 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:21:57,571 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:57,571 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:57,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:57,571 INFO [Thread-2354 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2354. 2024-12-02T14:21:57,572 DEBUG [Thread-2354 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:21:57,572 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:57,572 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:57,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:57,574 INFO [Thread-2346 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2346. 2024-12-02T14:21:57,574 DEBUG [Thread-2346 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:21:57,574 DEBUG [Thread-2346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:57,574 INFO [Thread-2350 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2350. 2024-12-02T14:21:57,574 DEBUG [Thread-2346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:57,574 DEBUG [Thread-2350 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:21:57,574 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:57,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:57,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:57,574 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T14:21:57,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T14:21:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:57,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:57,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:57,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149377698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:57,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:57,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:57,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:57,741 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:57,748 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202dc18c03745614de9bfc91088a8d85c83_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dc18c03745614de9bfc91088a8d85c83_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:57,750 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/6e1c5fa883a0449fb182125468964ee1, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:57,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/6e1c5fa883a0449fb182125468964ee1 is 175, key is test_row_0/A:col10/1733149317328/Put/seqid=0 2024-12-02T14:21:57,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742436_1612 (size=31255) 2024-12-02T14:21:57,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T14:21:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:57,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:57,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T14:21:57,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T14:21:57,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:57,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:57,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:57,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:57,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:57,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:58,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149378008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:58,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:58,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:58,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:58,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T14:21:58,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:58,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:58,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:58,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T14:21:58,159 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/6e1c5fa883a0449fb182125468964ee1 2024-12-02T14:21:58,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/c8f53c1253204621bcba8cd79c8c4baa is 50, key is test_row_0/B:col10/1733149317328/Put/seqid=0 2024-12-02T14:21:58,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742437_1613 (size=12301) 2024-12-02T14:21:58,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T14:21:58,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:58,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:58,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:58,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T14:21:58,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:58,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:58,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:58,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52968 deadline: 1733149378519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:58,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:58,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:58,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/c8f53c1253204621bcba8cd79c8c4baa 2024-12-02T14:21:58,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/6669dd9767ce411d975beaa97915d9fd is 50, key is test_row_0/C:col10/1733149317328/Put/seqid=0 2024-12-02T14:21:58,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742438_1614 (size=12301) 2024-12-02T14:21:58,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T14:21:58,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:58,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:58,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:58,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T14:21:58,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T14:21:58,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:58,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:58,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:58,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T14:21:58,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:58,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:58,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:58,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:58,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:58,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52934 deadline: 1733149378979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:21:58,981 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a3a61c9ba14f,46793,1733149138543 for 'test_row_2' in 1a7e8a8b8e1f978fa1c5a7606b30770c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 , the old value is region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:58,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:58,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., hostname=a3a61c9ba14f,46793,1733149138543, seqNum=5 because the exception is null or not the one we care about 2024-12-02T14:21:58,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/6669dd9767ce411d975beaa97915d9fd 2024-12-02T14:21:58,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/6e1c5fa883a0449fb182125468964ee1 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/6e1c5fa883a0449fb182125468964ee1 2024-12-02T14:21:59,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/6e1c5fa883a0449fb182125468964ee1, entries=150, sequenceid=313, filesize=30.5 K 2024-12-02T14:21:59,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/c8f53c1253204621bcba8cd79c8c4baa as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/c8f53c1253204621bcba8cd79c8c4baa 2024-12-02T14:21:59,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/c8f53c1253204621bcba8cd79c8c4baa, entries=150, sequenceid=313, filesize=12.0 K 2024-12-02T14:21:59,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/6669dd9767ce411d975beaa97915d9fd as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/6669dd9767ce411d975beaa97915d9fd 2024-12-02T14:21:59,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/6669dd9767ce411d975beaa97915d9fd, entries=150, sequenceid=313, filesize=12.0 K 2024-12-02T14:21:59,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 1681ms, sequenceid=313, compaction requested=true 2024-12-02T14:21:59,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:59,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T14:21:59,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:59,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T14:21:59,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:59,011 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:59,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a7e8a8b8e1f978fa1c5a7606b30770c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T14:21:59,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:59,011 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:59,012 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:59,012 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:59,012 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/A is initiating minor compaction (all files) 2024-12-02T14:21:59,012 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/B is initiating minor compaction (all files) 2024-12-02T14:21:59,012 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/B in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:59,012 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/A in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:59,012 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f8163baf63684781be6a914ca019985f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/0874de10c50d44e4be0161db47f6f37b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/c8f53c1253204621bcba8cd79c8c4baa] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=36.7 K 2024-12-02T14:21:59,012 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8c94e6672b514dddacfb1b88229508c4, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/b8838c8456d14cd589097bbd06d60a9e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/6e1c5fa883a0449fb182125468964ee1] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=92.2 K 2024-12-02T14:21:59,012 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:59,012 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. files: [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8c94e6672b514dddacfb1b88229508c4, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/b8838c8456d14cd589097bbd06d60a9e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/6e1c5fa883a0449fb182125468964ee1] 2024-12-02T14:21:59,012 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting f8163baf63684781be6a914ca019985f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733149314088 2024-12-02T14:21:59,012 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c94e6672b514dddacfb1b88229508c4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733149314088 2024-12-02T14:21:59,013 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 0874de10c50d44e4be0161db47f6f37b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733149314752 2024-12-02T14:21:59,013 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8838c8456d14cd589097bbd06d60a9e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733149314752 2024-12-02T14:21:59,013 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting c8f53c1253204621bcba8cd79c8c4baa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733149316698 2024-12-02T14:21:59,013 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e1c5fa883a0449fb182125468964ee1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733149316698 2024-12-02T14:21:59,021 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:59,022 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#B#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:59,022 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/32a44c3c324240f1aad17200767794b5 is 50, key is test_row_0/B:col10/1733149317328/Put/seqid=0 2024-12-02T14:21:59,024 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202d73d346637a344138b39931d0de9758c_1a7e8a8b8e1f978fa1c5a7606b30770c store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:59,027 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202d73d346637a344138b39931d0de9758c_1a7e8a8b8e1f978fa1c5a7606b30770c, store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:59,027 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d73d346637a344138b39931d0de9758c_1a7e8a8b8e1f978fa1c5a7606b30770c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:59,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742439_1615 (size=13085) 2024-12-02T14:21:59,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742440_1616 (size=4469) 2024-12-02T14:21:59,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46793 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T14:21:59,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:59,064 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T14:21:59,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:21:59,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:59,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:21:59,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:59,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:21:59,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:21:59,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027a089666d8f64604bbbd56aa3b63db10_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149317358/Put/seqid=0 2024-12-02T14:21:59,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742441_1617 (size=12454) 2024-12-02T14:21:59,433 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#A#compaction#514 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:59,435 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/b5f8b67e730147c0989dafb6e4e9944a is 175, key is test_row_0/A:col10/1733149317328/Put/seqid=0 2024-12-02T14:21:59,440 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/32a44c3c324240f1aad17200767794b5 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/32a44c3c324240f1aad17200767794b5 2024-12-02T14:21:59,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742442_1618 (size=32039) 2024-12-02T14:21:59,444 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/B of 1a7e8a8b8e1f978fa1c5a7606b30770c into 32a44c3c324240f1aad17200767794b5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:59,444 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:59,444 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/B, priority=13, startTime=1733149319011; duration=0sec 2024-12-02T14:21:59,445 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T14:21:59,445 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:B 2024-12-02T14:21:59,445 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T14:21:59,445 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T14:21:59,445 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1541): 1a7e8a8b8e1f978fa1c5a7606b30770c/C is initiating minor compaction (all files) 2024-12-02T14:21:59,445 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a7e8a8b8e1f978fa1c5a7606b30770c/C in TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:21:59,445 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/791d79d3d1df45b28ad6d95f970806cd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/4b0c54ccae8147679ccbb0a7ef1309ac, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/6669dd9767ce411d975beaa97915d9fd] into tmpdir=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp, totalSize=36.7 K 2024-12-02T14:21:59,446 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 791d79d3d1df45b28ad6d95f970806cd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733149314088 2024-12-02T14:21:59,446 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b0c54ccae8147679ccbb0a7ef1309ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733149314752 2024-12-02T14:21:59,446 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] compactions.Compactor(225): Compacting 6669dd9767ce411d975beaa97915d9fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733149316698 2024-12-02T14:21:59,451 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a7e8a8b8e1f978fa1c5a7606b30770c#C#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T14:21:59,452 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/a0ebdede0591479283686fe0bc1620b9 is 50, key is test_row_0/C:col10/1733149317328/Put/seqid=0 2024-12-02T14:21:59,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742443_1619 (size=13085) 2024-12-02T14:21:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:21:59,478 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027a089666d8f64604bbbd56aa3b63db10_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027a089666d8f64604bbbd56aa3b63db10_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:59,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/d4a48d2d02ab4588b25d58f91755ba7a, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:21:59,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/d4a48d2d02ab4588b25d58f91755ba7a is 175, key is test_row_0/A:col10/1733149317358/Put/seqid=0 2024-12-02T14:21:59,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742444_1620 (size=31255) 2024-12-02T14:21:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46793 {}] regionserver.HRegion(8855): Flush requested on 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:21:59,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. as already flushing 2024-12-02T14:21:59,530 INFO [Thread-2343 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2343. 2024-12-02T14:21:59,530 DEBUG [Thread-2343 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:21:59,530 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:59,530 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:59,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T14:21:59,853 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/b5f8b67e730147c0989dafb6e4e9944a as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/b5f8b67e730147c0989dafb6e4e9944a 2024-12-02T14:21:59,858 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/A of 1a7e8a8b8e1f978fa1c5a7606b30770c into b5f8b67e730147c0989dafb6e4e9944a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:59,858 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:59,859 INFO [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/A, priority=13, startTime=1733149319011; duration=0sec 2024-12-02T14:21:59,859 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:59,859 DEBUG [RS:0;a3a61c9ba14f:46793-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:A 2024-12-02T14:21:59,860 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/a0ebdede0591479283686fe0bc1620b9 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/a0ebdede0591479283686fe0bc1620b9 2024-12-02T14:21:59,865 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a7e8a8b8e1f978fa1c5a7606b30770c/C of 1a7e8a8b8e1f978fa1c5a7606b30770c into a0ebdede0591479283686fe0bc1620b9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T14:21:59,865 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:21:59,865 INFO [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., storeName=1a7e8a8b8e1f978fa1c5a7606b30770c/C, priority=13, startTime=1733149319011; duration=0sec 2024-12-02T14:21:59,865 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T14:21:59,865 DEBUG [RS:0;a3a61c9ba14f:46793-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a7e8a8b8e1f978fa1c5a7606b30770c:C 2024-12-02T14:21:59,882 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/d4a48d2d02ab4588b25d58f91755ba7a 2024-12-02T14:21:59,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/4e60f59f9dc4411c8c3436cb823eb932 is 50, key is test_row_0/B:col10/1733149317358/Put/seqid=0 2024-12-02T14:21:59,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742445_1621 (size=12301) 2024-12-02T14:22:00,292 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/4e60f59f9dc4411c8c3436cb823eb932 2024-12-02T14:22:00,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/48b83ba2cd054151ba01f30e39a63c48 is 50, key is test_row_0/C:col10/1733149317358/Put/seqid=0 2024-12-02T14:22:00,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742446_1622 (size=12301) 2024-12-02T14:22:00,707 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/48b83ba2cd054151ba01f30e39a63c48 2024-12-02T14:22:00,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/d4a48d2d02ab4588b25d58f91755ba7a as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/d4a48d2d02ab4588b25d58f91755ba7a 2024-12-02T14:22:00,715 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/d4a48d2d02ab4588b25d58f91755ba7a, entries=150, sequenceid=327, filesize=30.5 K 2024-12-02T14:22:00,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/4e60f59f9dc4411c8c3436cb823eb932 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/4e60f59f9dc4411c8c3436cb823eb932 2024-12-02T14:22:00,720 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/4e60f59f9dc4411c8c3436cb823eb932, entries=150, sequenceid=327, filesize=12.0 K 2024-12-02T14:22:00,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/48b83ba2cd054151ba01f30e39a63c48 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/48b83ba2cd054151ba01f30e39a63c48 2024-12-02T14:22:00,724 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/48b83ba2cd054151ba01f30e39a63c48, entries=150, sequenceid=327, filesize=12.0 K 2024-12-02T14:22:00,724 INFO [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=6.71 KB/6870 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 1660ms, sequenceid=327, compaction requested=false 2024-12-02T14:22:00,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: 2024-12-02T14:22:00,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:22:00,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-02T14:22:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-02T14:22:00,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-02T14:22:00,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2150 sec 2024-12-02T14:22:00,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 3.2190 sec 2024-12-02T14:22:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T14:22:01,690 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:22:02,993 INFO [Thread-2339 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2339. 2024-12-02T14:22:02,993 DEBUG [Thread-2339 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:22:02,993 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:22:02,993 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:22:02,993 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:22:03,162 INFO [Thread-2341 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2341. 2024-12-02T14:22:03,162 DEBUG [Thread-2341 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:22:03,162 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:22:03,162 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:22:03,162 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:22:03,213 INFO [Thread-2335 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2335. 2024-12-02T14:22:03,213 DEBUG [Thread-2335 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:22:03,214 DEBUG [Thread-2335 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:22:03,214 DEBUG [Thread-2335 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:22:03,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:22:03,221 INFO [Thread-2337 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2337. 2024-12-02T14:22:03,222 DEBUG [Thread-2337 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T14:22:03,222 DEBUG [Thread-2337 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:22:03,222 DEBUG [Thread-2337 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:22:03,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:22:03,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T14:22:03,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-12-02T14:22:03,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-12-02T14:22:03,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-12-02T14:22:03,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-12-02T14:22:03,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-12-02T14:22:03,222 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T14:22:03,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4246 2024-12-02T14:22:03,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4198 2024-12-02T14:22:03,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4247 2024-12-02T14:22:03,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4243 2024-12-02T14:22:03,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4186 2024-12-02T14:22:03,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T14:22:03,223 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T14:22:03,223 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T14:22:03,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:22:03,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:22:03,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:22:03,225 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T14:22:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T14:22:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T14:22:03,230 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149323230"}]},"ts":"1733149323230"} 2024-12-02T14:22:03,232 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T14:22:03,233 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T14:22:03,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T14:22:03,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a7e8a8b8e1f978fa1c5a7606b30770c, UNASSIGN}] 2024-12-02T14:22:03,237 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a7e8a8b8e1f978fa1c5a7606b30770c, UNASSIGN 2024-12-02T14:22:03,238 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=1a7e8a8b8e1f978fa1c5a7606b30770c, regionState=CLOSING, regionLocation=a3a61c9ba14f,46793,1733149138543 2024-12-02T14:22:03,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a7e8a8b8e1f978fa1c5a7606b30770c, UNASSIGN because future has completed 2024-12-02T14:22:03,240 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T14:22:03,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543}] 2024-12-02T14:22:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T14:22:03,394 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:03,394 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T14:22:03,394 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing 1a7e8a8b8e1f978fa1c5a7606b30770c, disabling compactions & flushes 2024-12-02T14:22:03,394 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:22:03,395 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:22:03,395 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. after waiting 0 ms 2024-12-02T14:22:03,395 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:22:03,395 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T14:22:03,396 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=A 2024-12-02T14:22:03,396 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:22:03,396 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=B 2024-12-02T14:22:03,396 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:22:03,396 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a7e8a8b8e1f978fa1c5a7606b30770c, store=C 2024-12-02T14:22:03,396 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T14:22:03,406 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202046f9861634e49e2a11869f1711b1d2a_1a7e8a8b8e1f978fa1c5a7606b30770c is 50, key is test_row_0/A:col10/1733149323159/Put/seqid=0 2024-12-02T14:22:03,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742447_1623 (size=9914) 2024-12-02T14:22:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T14:22:03,810 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:22:03,818 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202046f9861634e49e2a11869f1711b1d2a_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202046f9861634e49e2a11869f1711b1d2a_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:03,819 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/540ed95e5ef1408a8e9cdf7743e9e7e5, store: [table=TestAcidGuarantees family=A region=1a7e8a8b8e1f978fa1c5a7606b30770c] 2024-12-02T14:22:03,820 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/540ed95e5ef1408a8e9cdf7743e9e7e5 is 175, key is test_row_0/A:col10/1733149323159/Put/seqid=0 2024-12-02T14:22:03,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742448_1624 (size=22561) 2024-12-02T14:22:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T14:22:04,224 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/540ed95e5ef1408a8e9cdf7743e9e7e5 2024-12-02T14:22:04,235 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/35c04a22081d46bcaf1c5488abc848a0 is 50, key is test_row_0/B:col10/1733149323159/Put/seqid=0 2024-12-02T14:22:04,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742449_1625 (size=9857) 2024-12-02T14:22:04,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T14:22:04,640 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/35c04a22081d46bcaf1c5488abc848a0 2024-12-02T14:22:04,652 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/be7c784c31b64db0999df8f49a48a6a4 is 50, key is test_row_0/C:col10/1733149323159/Put/seqid=0 2024-12-02T14:22:04,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742450_1626 (size=9857) 2024-12-02T14:22:05,056 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/be7c784c31b64db0999df8f49a48a6a4 2024-12-02T14:22:05,067 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/A/540ed95e5ef1408a8e9cdf7743e9e7e5 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/540ed95e5ef1408a8e9cdf7743e9e7e5 2024-12-02T14:22:05,073 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/540ed95e5ef1408a8e9cdf7743e9e7e5, entries=100, sequenceid=338, filesize=22.0 K 2024-12-02T14:22:05,074 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/B/35c04a22081d46bcaf1c5488abc848a0 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/35c04a22081d46bcaf1c5488abc848a0 2024-12-02T14:22:05,078 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/35c04a22081d46bcaf1c5488abc848a0, entries=100, sequenceid=338, filesize=9.6 K 2024-12-02T14:22:05,079 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/.tmp/C/be7c784c31b64db0999df8f49a48a6a4 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/be7c784c31b64db0999df8f49a48a6a4 2024-12-02T14:22:05,083 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/be7c784c31b64db0999df8f49a48a6a4, entries=100, sequenceid=338, filesize=9.6 K 2024-12-02T14:22:05,084 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 1689ms, sequenceid=338, compaction requested=true 2024-12-02T14:22:05,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/221b04bd07ff45399afab9e9fe2d2055, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/06b0c8bd65f8428683d2679321c0dc3e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8ab458bcf33048e0b579113acfb68e1c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/13cc40aa13264fdc96491b39c407bb24, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3a597ce206784e44b9a6f1fcfc9e4108, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/cb03941b1fff47419eef69a868b92889, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/0bbd94392f6144b184980fee4af8fc33, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3de31fa980734d6aa3ac0313eb1be0b7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/d65e40dba4eb4a6e845f65d280c14042, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/93f0aef667cd4c4a8d91c68d6689e5fb, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/0e15e1fdeb8942eb8ccdc20f0eb91e96, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/5769c69e363c4d42beed5c04a5b256b0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3f8937e8508543e4a9827cb8ae191dbd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/e064bf70a43c40d4a2e46d24cd4e03cf, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/c23affedc9994c76a9ec06ad90d4fd80, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8518b7aef05540488f43c1e65ca01ed7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3581319b2db04ae68ab7163ec63be7c2, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/f46a62cde4b74affb003329735557012, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/e8d42fcabf3e40488516cf877e3e4621, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8c94e6672b514dddacfb1b88229508c4, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/f92b9f69e60840d7b2ef77e1fe67d221, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/b8838c8456d14cd589097bbd06d60a9e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/6e1c5fa883a0449fb182125468964ee1] to archive 2024-12-02T14:22:05,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:22:05,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/221b04bd07ff45399afab9e9fe2d2055 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/221b04bd07ff45399afab9e9fe2d2055 2024-12-02T14:22:05,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/06b0c8bd65f8428683d2679321c0dc3e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/06b0c8bd65f8428683d2679321c0dc3e 2024-12-02T14:22:05,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8ab458bcf33048e0b579113acfb68e1c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8ab458bcf33048e0b579113acfb68e1c 2024-12-02T14:22:05,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/13cc40aa13264fdc96491b39c407bb24 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/13cc40aa13264fdc96491b39c407bb24 2024-12-02T14:22:05,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3a597ce206784e44b9a6f1fcfc9e4108 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3a597ce206784e44b9a6f1fcfc9e4108 2024-12-02T14:22:05,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/cb03941b1fff47419eef69a868b92889 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/cb03941b1fff47419eef69a868b92889 2024-12-02T14:22:05,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/0bbd94392f6144b184980fee4af8fc33 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/0bbd94392f6144b184980fee4af8fc33 2024-12-02T14:22:05,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3de31fa980734d6aa3ac0313eb1be0b7 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3de31fa980734d6aa3ac0313eb1be0b7 2024-12-02T14:22:05,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/d65e40dba4eb4a6e845f65d280c14042 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/d65e40dba4eb4a6e845f65d280c14042 2024-12-02T14:22:05,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/93f0aef667cd4c4a8d91c68d6689e5fb to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/93f0aef667cd4c4a8d91c68d6689e5fb 2024-12-02T14:22:05,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/0e15e1fdeb8942eb8ccdc20f0eb91e96 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/0e15e1fdeb8942eb8ccdc20f0eb91e96 2024-12-02T14:22:05,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/5769c69e363c4d42beed5c04a5b256b0 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/5769c69e363c4d42beed5c04a5b256b0 2024-12-02T14:22:05,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3f8937e8508543e4a9827cb8ae191dbd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3f8937e8508543e4a9827cb8ae191dbd 2024-12-02T14:22:05,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/e064bf70a43c40d4a2e46d24cd4e03cf to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/e064bf70a43c40d4a2e46d24cd4e03cf 2024-12-02T14:22:05,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/c23affedc9994c76a9ec06ad90d4fd80 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/c23affedc9994c76a9ec06ad90d4fd80 2024-12-02T14:22:05,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8518b7aef05540488f43c1e65ca01ed7 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8518b7aef05540488f43c1e65ca01ed7 2024-12-02T14:22:05,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3581319b2db04ae68ab7163ec63be7c2 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/3581319b2db04ae68ab7163ec63be7c2 2024-12-02T14:22:05,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/f46a62cde4b74affb003329735557012 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/f46a62cde4b74affb003329735557012 2024-12-02T14:22:05,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/e8d42fcabf3e40488516cf877e3e4621 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/e8d42fcabf3e40488516cf877e3e4621 2024-12-02T14:22:05,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8c94e6672b514dddacfb1b88229508c4 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/8c94e6672b514dddacfb1b88229508c4 2024-12-02T14:22:05,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/f92b9f69e60840d7b2ef77e1fe67d221 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/f92b9f69e60840d7b2ef77e1fe67d221 2024-12-02T14:22:05,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/b8838c8456d14cd589097bbd06d60a9e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/b8838c8456d14cd589097bbd06d60a9e 2024-12-02T14:22:05,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/6e1c5fa883a0449fb182125468964ee1 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/6e1c5fa883a0449fb182125468964ee1 2024-12-02T14:22:05,106 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/4f554487536a4a3ca5c094b4b7f5242f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/256934c23da04e74a68087073c5ef600, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/cc81053ee1b0446cb490bd08b3fb6526, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/ef47df58af8b43cb9555f44650901501, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/ae75ed02155642a98557033d1b5432c2, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7e1e17b740a5435687479fd683358a25, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/25ca66f01fd242e595d1ccd84d6cbf69, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/b03c7981ff2d42488339bf4d28e2a3ae, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/5bec31690b1c4a148ad8aadbcfcf40f8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/b814e625b10a40f7a97651d9dfe63d1a, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f5feefac6a104ccfaa78471fbda711b7, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f764305b01aa45fda90683acf2a45551, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/2eb3149f62624731bc729bc241ed5b58, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/87e35e5955ec41ec8759b2ed470e14fc, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/9af8f0e30d564021bcf4d78268ea49b5, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/2e48869eac5a46fa8143fcbbb4b050e8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/38cf5f1bec554d44b8d5fde3161fa3c0, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7be1eeeeaf1a4e389a677623d129db82, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/9f3b23a99a7545418565c05c7443542f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f8163baf63684781be6a914ca019985f, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7070bff9ccc046d6a78aed6eb355deee, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/0874de10c50d44e4be0161db47f6f37b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/c8f53c1253204621bcba8cd79c8c4baa] to archive 2024-12-02T14:22:05,107 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:22:05,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/4f554487536a4a3ca5c094b4b7f5242f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/4f554487536a4a3ca5c094b4b7f5242f 2024-12-02T14:22:05,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/256934c23da04e74a68087073c5ef600 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/256934c23da04e74a68087073c5ef600 2024-12-02T14:22:05,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/cc81053ee1b0446cb490bd08b3fb6526 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/cc81053ee1b0446cb490bd08b3fb6526 2024-12-02T14:22:05,110 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/ef47df58af8b43cb9555f44650901501 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/ef47df58af8b43cb9555f44650901501 2024-12-02T14:22:05,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/ae75ed02155642a98557033d1b5432c2 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/ae75ed02155642a98557033d1b5432c2 2024-12-02T14:22:05,112 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7e1e17b740a5435687479fd683358a25 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7e1e17b740a5435687479fd683358a25 2024-12-02T14:22:05,112 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/25ca66f01fd242e595d1ccd84d6cbf69 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/25ca66f01fd242e595d1ccd84d6cbf69 2024-12-02T14:22:05,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/b03c7981ff2d42488339bf4d28e2a3ae to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/b03c7981ff2d42488339bf4d28e2a3ae 2024-12-02T14:22:05,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/5bec31690b1c4a148ad8aadbcfcf40f8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/5bec31690b1c4a148ad8aadbcfcf40f8 2024-12-02T14:22:05,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/b814e625b10a40f7a97651d9dfe63d1a to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/b814e625b10a40f7a97651d9dfe63d1a 2024-12-02T14:22:05,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f5feefac6a104ccfaa78471fbda711b7 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f5feefac6a104ccfaa78471fbda711b7 2024-12-02T14:22:05,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f764305b01aa45fda90683acf2a45551 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f764305b01aa45fda90683acf2a45551 2024-12-02T14:22:05,116 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/2eb3149f62624731bc729bc241ed5b58 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/2eb3149f62624731bc729bc241ed5b58 2024-12-02T14:22:05,116 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/87e35e5955ec41ec8759b2ed470e14fc to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/87e35e5955ec41ec8759b2ed470e14fc 2024-12-02T14:22:05,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/9af8f0e30d564021bcf4d78268ea49b5 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/9af8f0e30d564021bcf4d78268ea49b5 2024-12-02T14:22:05,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/2e48869eac5a46fa8143fcbbb4b050e8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/2e48869eac5a46fa8143fcbbb4b050e8 2024-12-02T14:22:05,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/38cf5f1bec554d44b8d5fde3161fa3c0 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/38cf5f1bec554d44b8d5fde3161fa3c0 2024-12-02T14:22:05,119 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7be1eeeeaf1a4e389a677623d129db82 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7be1eeeeaf1a4e389a677623d129db82 2024-12-02T14:22:05,119 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/9f3b23a99a7545418565c05c7443542f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/9f3b23a99a7545418565c05c7443542f 2024-12-02T14:22:05,120 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f8163baf63684781be6a914ca019985f to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/f8163baf63684781be6a914ca019985f 2024-12-02T14:22:05,121 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7070bff9ccc046d6a78aed6eb355deee to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/7070bff9ccc046d6a78aed6eb355deee 2024-12-02T14:22:05,121 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/0874de10c50d44e4be0161db47f6f37b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/0874de10c50d44e4be0161db47f6f37b 2024-12-02T14:22:05,122 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/c8f53c1253204621bcba8cd79c8c4baa to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/c8f53c1253204621bcba8cd79c8c4baa 2024-12-02T14:22:05,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/9302fc94d21844cc9cdca423e7cd7c96, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/31948a4abfc24b00ac9657bdd85a38a9, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/9f833d8de7624989bd9a1d87628d1ca3, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/6d8f311baab241a2b3298bf7706c8b19, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/3c375ef711e84755a4ac8fd8ccf434fc, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/b765e9c08d0c481190e3595ff0059a3b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/d6901758c80c4192a8459d759a52a4fe, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/688d81e8125f4ce2ac89e35c770220b1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/5b047de7de004f97895f1d529220261e, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/7fbeb3bcabcf46e58a7f23cdd3768f61, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/14b9656bf6eb4ac59c5a0224e5b1fb5d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/82efd7708fb04ea2b2bef9e2116a43b8, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/49b91def1549447b9f45cd083dd1171d, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/48247ddffff34262b916943711271d13, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/0b45d6c0bd304f9d8642a9b53a267576, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/711d30f4631a4002adcdec65c291cc6b, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/c638c9a79d8745be99b0bbe072e199d1, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/464780a8a4dc4d9e9cacf7ec23f0dbce, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/33919bfd70e84ca5bd8d6fa29b108566, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/791d79d3d1df45b28ad6d95f970806cd, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/948ebafd448d4f6a9270589ff3dc6e8c, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/4b0c54ccae8147679ccbb0a7ef1309ac, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/6669dd9767ce411d975beaa97915d9fd] to archive 2024-12-02T14:22:05,124 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T14:22:05,124 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/9302fc94d21844cc9cdca423e7cd7c96 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/9302fc94d21844cc9cdca423e7cd7c96 2024-12-02T14:22:05,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/31948a4abfc24b00ac9657bdd85a38a9 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/31948a4abfc24b00ac9657bdd85a38a9 2024-12-02T14:22:05,126 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/9f833d8de7624989bd9a1d87628d1ca3 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/9f833d8de7624989bd9a1d87628d1ca3 2024-12-02T14:22:05,126 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/6d8f311baab241a2b3298bf7706c8b19 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/6d8f311baab241a2b3298bf7706c8b19 2024-12-02T14:22:05,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/3c375ef711e84755a4ac8fd8ccf434fc to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/3c375ef711e84755a4ac8fd8ccf434fc 2024-12-02T14:22:05,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/b765e9c08d0c481190e3595ff0059a3b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/b765e9c08d0c481190e3595ff0059a3b 2024-12-02T14:22:05,128 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/d6901758c80c4192a8459d759a52a4fe to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/d6901758c80c4192a8459d759a52a4fe 2024-12-02T14:22:05,129 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/688d81e8125f4ce2ac89e35c770220b1 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/688d81e8125f4ce2ac89e35c770220b1 2024-12-02T14:22:05,130 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/5b047de7de004f97895f1d529220261e to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/5b047de7de004f97895f1d529220261e 2024-12-02T14:22:05,131 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/7fbeb3bcabcf46e58a7f23cdd3768f61 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/7fbeb3bcabcf46e58a7f23cdd3768f61 2024-12-02T14:22:05,131 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/14b9656bf6eb4ac59c5a0224e5b1fb5d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/14b9656bf6eb4ac59c5a0224e5b1fb5d 2024-12-02T14:22:05,132 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/82efd7708fb04ea2b2bef9e2116a43b8 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/82efd7708fb04ea2b2bef9e2116a43b8 2024-12-02T14:22:05,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/49b91def1549447b9f45cd083dd1171d to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/49b91def1549447b9f45cd083dd1171d 2024-12-02T14:22:05,134 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/48247ddffff34262b916943711271d13 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/48247ddffff34262b916943711271d13 2024-12-02T14:22:05,135 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/0b45d6c0bd304f9d8642a9b53a267576 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/0b45d6c0bd304f9d8642a9b53a267576 2024-12-02T14:22:05,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/711d30f4631a4002adcdec65c291cc6b to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/711d30f4631a4002adcdec65c291cc6b 2024-12-02T14:22:05,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/c638c9a79d8745be99b0bbe072e199d1 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/c638c9a79d8745be99b0bbe072e199d1 2024-12-02T14:22:05,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/464780a8a4dc4d9e9cacf7ec23f0dbce to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/464780a8a4dc4d9e9cacf7ec23f0dbce 2024-12-02T14:22:05,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/33919bfd70e84ca5bd8d6fa29b108566 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/33919bfd70e84ca5bd8d6fa29b108566 2024-12-02T14:22:05,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/791d79d3d1df45b28ad6d95f970806cd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/791d79d3d1df45b28ad6d95f970806cd 2024-12-02T14:22:05,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/948ebafd448d4f6a9270589ff3dc6e8c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/948ebafd448d4f6a9270589ff3dc6e8c 2024-12-02T14:22:05,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/4b0c54ccae8147679ccbb0a7ef1309ac to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/4b0c54ccae8147679ccbb0a7ef1309ac 2024-12-02T14:22:05,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/6669dd9767ce411d975beaa97915d9fd to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/6669dd9767ce411d975beaa97915d9fd 2024-12-02T14:22:05,145 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/recovered.edits/341.seqid, newMaxSeqId=341, maxSeqId=4 2024-12-02T14:22:05,145 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. 2024-12-02T14:22:05,145 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for 1a7e8a8b8e1f978fa1c5a7606b30770c: Waiting for close lock at 1733149323394Running coprocessor pre-close hooks at 1733149323394Disabling compacts and flushes for region at 1733149323394Disabling writes for close at 1733149323395 (+1 ms)Obtaining lock to block concurrent updates at 1733149323395Preparing flush snapshotting stores in 1a7e8a8b8e1f978fa1c5a7606b30770c at 1733149323395Finished memstore snapshotting TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733149323396 (+1 ms)Flushing stores of TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c. at 1733149323397 (+1 ms)Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c/A: creating writer at 1733149323397Flushing mob file 1a7e8a8b8e1f978fa1c5a7606b30770c/A: appending metadata at 1733149323406 (+9 ms)Flushing mob file 1a7e8a8b8e1f978fa1c5a7606b30770c/A: closing flushed file at 1733149323406Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c/A: appending metadata at 1733149323819 (+413 ms)Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c/A: closing flushed file at 1733149323820 (+1 ms)Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c/B: creating writer at 1733149324230 (+410 ms)Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c/B: appending metadata at 1733149324234 (+4 ms)Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c/B: closing flushed file at 1733149324234Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c/C: creating writer at 1733149324649 (+415 ms)Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c/C: appending metadata at 1733149324651 (+2 ms)Flushing 1a7e8a8b8e1f978fa1c5a7606b30770c/C: closing flushed file at 1733149324651Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b77101: reopening flushed file at 1733149325066 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71ea0e5d: reopening flushed file at 1733149325073 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24a8ab32: reopening flushed file at 1733149325078 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1a7e8a8b8e1f978fa1c5a7606b30770c in 1689ms, sequenceid=338, compaction requested=true at 1733149325084 (+6 ms)Writing region close event to WAL at 1733149325142 (+58 ms)Running coprocessor post-close hooks at 1733149325145 (+3 ms)Closed at 1733149325145 2024-12-02T14:22:05,147 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed 1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,147 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=1a7e8a8b8e1f978fa1c5a7606b30770c, regionState=CLOSED 2024-12-02T14:22:05,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 because future has completed 2024-12-02T14:22:05,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-02T14:22:05,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure 1a7e8a8b8e1f978fa1c5a7606b30770c, server=a3a61c9ba14f,46793,1733149138543 in 1.9090 sec 2024-12-02T14:22:05,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-12-02T14:22:05,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a7e8a8b8e1f978fa1c5a7606b30770c, UNASSIGN in 1.9160 sec 2024-12-02T14:22:05,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-02T14:22:05,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9210 sec 2024-12-02T14:22:05,157 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149325157"}]},"ts":"1733149325157"} 2024-12-02T14:22:05,158 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T14:22:05,158 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T14:22:05,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9340 sec 2024-12-02T14:22:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T14:22:05,380 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:22:05,381 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T14:22:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:22:05,385 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:22:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T14:22:05,388 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:22:05,391 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,394 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C, FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/recovered.edits] 2024-12-02T14:22:05,397 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/540ed95e5ef1408a8e9cdf7743e9e7e5 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/540ed95e5ef1408a8e9cdf7743e9e7e5 2024-12-02T14:22:05,398 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/b5f8b67e730147c0989dafb6e4e9944a to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/b5f8b67e730147c0989dafb6e4e9944a 2024-12-02T14:22:05,400 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/d4a48d2d02ab4588b25d58f91755ba7a to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/A/d4a48d2d02ab4588b25d58f91755ba7a 2024-12-02T14:22:05,403 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/32a44c3c324240f1aad17200767794b5 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/32a44c3c324240f1aad17200767794b5 2024-12-02T14:22:05,405 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/35c04a22081d46bcaf1c5488abc848a0 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/35c04a22081d46bcaf1c5488abc848a0 2024-12-02T14:22:05,406 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/4e60f59f9dc4411c8c3436cb823eb932 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/B/4e60f59f9dc4411c8c3436cb823eb932 2024-12-02T14:22:05,409 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/48b83ba2cd054151ba01f30e39a63c48 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/48b83ba2cd054151ba01f30e39a63c48 2024-12-02T14:22:05,411 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/a0ebdede0591479283686fe0bc1620b9 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/a0ebdede0591479283686fe0bc1620b9 2024-12-02T14:22:05,413 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/be7c784c31b64db0999df8f49a48a6a4 to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/C/be7c784c31b64db0999df8f49a48a6a4 2024-12-02T14:22:05,416 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/recovered.edits/341.seqid to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c/recovered.edits/341.seqid 2024-12-02T14:22:05,417 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/default/TestAcidGuarantees/1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,417 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T14:22:05,417 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T14:22:05,418 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T14:22:05,422 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020438354ba5dc447dbff9c18cb69b0fb9_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020438354ba5dc447dbff9c18cb69b0fb9_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,424 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202046f9861634e49e2a11869f1711b1d2a_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202046f9861634e49e2a11869f1711b1d2a_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,426 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202321330aa2eb94eba8d51e524ca9e3fac_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202321330aa2eb94eba8d51e524ca9e3fac_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,428 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202350c3056624c4f75ac196ea883fd939c_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202350c3056624c4f75ac196ea883fd939c_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,429 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202354513869e204aed8f2b7044bafc2a19_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202354513869e204aed8f2b7044bafc2a19_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,430 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120239095b622f5640d2a475eded42e6efb0_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120239095b622f5640d2a475eded42e6efb0_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,431 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024778ec5173ef494985ab3055e749084d_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024778ec5173ef494985ab3055e749084d_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,432 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024821679a86974334806faa7e820dcec3_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024821679a86974334806faa7e820dcec3_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,434 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202602a1f83563844b5be32ce84d520697d_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202602a1f83563844b5be32ce84d520697d_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,435 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026815bee4dbfd444b9466f84bb182eb85_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026815bee4dbfd444b9466f84bb182eb85_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,436 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027a089666d8f64604bbbd56aa3b63db10_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027a089666d8f64604bbbd56aa3b63db10_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,437 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027a5103bddca647a485b6c327d945c3c8_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027a5103bddca647a485b6c327d945c3c8_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,438 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027afd0e5d20084781a8f3027932cf18b1_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027afd0e5d20084781a8f3027932cf18b1_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,440 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029641599ce7254ce28ddaae1a3ffcaa37_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029641599ce7254ce28ddaae1a3ffcaa37_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,441 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a27ffe52392b4fb7914dc7bf860b9e2e_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a27ffe52392b4fb7914dc7bf860b9e2e_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,442 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b71d4229e8794fd7b98d0b9d3c505377_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b71d4229e8794fd7b98d0b9d3c505377_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,443 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d33962a2e15d460a80b6cb09e6bea135_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d33962a2e15d460a80b6cb09e6bea135_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,444 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dbdb0f8bfaf749fca47baf334d2f13d8_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dbdb0f8bfaf749fca47baf334d2f13d8_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,445 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dc18c03745614de9bfc91088a8d85c83_1a7e8a8b8e1f978fa1c5a7606b30770c to hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dc18c03745614de9bfc91088a8d85c83_1a7e8a8b8e1f978fa1c5a7606b30770c 2024-12-02T14:22:05,446 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T14:22:05,447 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:22:05,449 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T14:22:05,450 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T14:22:05,451 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:22:05,451 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T14:22:05,452 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733149325451"}]},"ts":"9223372036854775807"} 2024-12-02T14:22:05,453 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T14:22:05,453 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1a7e8a8b8e1f978fa1c5a7606b30770c, NAME => 'TestAcidGuarantees,,1733149294423.1a7e8a8b8e1f978fa1c5a7606b30770c.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T14:22:05,453 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T14:22:05,453 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733149325453"}]},"ts":"9223372036854775807"} 2024-12-02T14:22:05,454 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T14:22:05,455 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T14:22:05,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 74 msec 2024-12-02T14:22:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35877 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T14:22:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T14:22:05,499 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T14:22:05,508 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246 (was 245) - Thread LEAK? -, OpenFileDescriptor=447 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=377 (was 377), ProcessCount=11 (was 11), AvailableMemoryMB=6219 (was 6628) 2024-12-02T14:22:05,508 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T14:22:05,509 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T14:22:05,509 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T14:22:05,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:22:05,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:22:05,509 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:22:05,509 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T14:22:05,509 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=261653369, stopped=false 2024-12-02T14:22:05,510 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a3a61c9ba14f,35877,1733149137941 2024-12-02T14:22:05,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T14:22:05,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T14:22:05,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:22:05,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:22:05,512 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T14:22:05,513 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T14:22:05,513 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T14:22:05,513 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T14:22:05,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:22:05,513 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a3a61c9ba14f,46793,1733149138543' ***** 2024-12-02T14:22:05,513 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T14:22:05,514 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T14:22:05,514 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T14:22:05,514 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T14:22:05,514 INFO [RS:0;a3a61c9ba14f:46793 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T14:22:05,514 INFO [RS:0;a3a61c9ba14f:46793 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T14:22:05,514 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(959): stopping server a3a61c9ba14f,46793,1733149138543 2024-12-02T14:22:05,514 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T14:22:05,515 INFO [RS:0;a3a61c9ba14f:46793 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a3a61c9ba14f:46793. 2024-12-02T14:22:05,515 DEBUG [RS:0;a3a61c9ba14f:46793 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T14:22:05,515 DEBUG [RS:0;a3a61c9ba14f:46793 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:22:05,515 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T14:22:05,515 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T14:22:05,515 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T14:22:05,515 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T14:22:05,516 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-02T14:22:05,516 DEBUG [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-02T14:22:05,516 DEBUG [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T14:22:05,516 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T14:22:05,516 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T14:22:05,516 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T14:22:05,516 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T14:22:05,516 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T14:22:05,516 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-02T14:22:05,535 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/.tmp/info/ccdf463fbb484518bc782f960db4f640 is 95, key is TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8./info:/1733149168443/DeleteFamily/seqid=0 2024-12-02T14:22:05,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742451_1627 (size=6514) 2024-12-02T14:22:05,609 INFO [regionserver/a3a61c9ba14f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T14:22:05,716 DEBUG [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T14:22:05,916 DEBUG [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T14:22:05,940 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/.tmp/info/ccdf463fbb484518bc782f960db4f640 2024-12-02T14:22:05,963 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/.tmp/ns/4933d95eb974421b8dea7e8acd250803 is 93, key is TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8./ns:/1733149168397/DeleteFamily/seqid=0 2024-12-02T14:22:05,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742452_1628 (size=5906) 2024-12-02T14:22:06,098 INFO [regionserver/a3a61c9ba14f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T14:22:06,098 INFO [regionserver/a3a61c9ba14f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T14:22:06,117 DEBUG [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T14:22:06,317 DEBUG [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T14:22:06,367 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/.tmp/ns/4933d95eb974421b8dea7e8acd250803 2024-12-02T14:22:06,393 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/.tmp/rep_barrier/e7a5ea70ea294c93ae1079662f6b4310 is 102, key is TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8./rep_barrier:/1733149168397/DeleteFamily/seqid=0 2024-12-02T14:22:06,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742453_1629 (size=6025) 2024-12-02T14:22:06,517 DEBUG [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T14:22:06,717 DEBUG [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T14:22:06,798 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/.tmp/rep_barrier/e7a5ea70ea294c93ae1079662f6b4310 2024-12-02T14:22:06,825 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/.tmp/table/917e65516fcc4cbfa8dbcbec40c5d1b9 is 96, key is TestAcidGuarantees,,1733149141533.4eaa960f4ddde5e2082f5ac5eba8cec8./table:/1733149168397/DeleteFamily/seqid=0 2024-12-02T14:22:06,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742454_1630 (size=5936) 2024-12-02T14:22:06,918 DEBUG [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T14:22:07,118 DEBUG [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T14:22:07,229 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/.tmp/table/917e65516fcc4cbfa8dbcbec40c5d1b9 2024-12-02T14:22:07,239 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/.tmp/info/ccdf463fbb484518bc782f960db4f640 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/info/ccdf463fbb484518bc782f960db4f640 2024-12-02T14:22:07,243 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/info/ccdf463fbb484518bc782f960db4f640, entries=12, sequenceid=90, filesize=6.4 K 2024-12-02T14:22:07,245 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/.tmp/ns/4933d95eb974421b8dea7e8acd250803 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/ns/4933d95eb974421b8dea7e8acd250803 2024-12-02T14:22:07,249 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/ns/4933d95eb974421b8dea7e8acd250803, entries=8, sequenceid=90, filesize=5.8 K 2024-12-02T14:22:07,250 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/.tmp/rep_barrier/e7a5ea70ea294c93ae1079662f6b4310 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/rep_barrier/e7a5ea70ea294c93ae1079662f6b4310 2024-12-02T14:22:07,253 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/rep_barrier/e7a5ea70ea294c93ae1079662f6b4310, entries=6, sequenceid=90, filesize=5.9 K 2024-12-02T14:22:07,254 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/.tmp/table/917e65516fcc4cbfa8dbcbec40c5d1b9 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/table/917e65516fcc4cbfa8dbcbec40c5d1b9 2024-12-02T14:22:07,257 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/table/917e65516fcc4cbfa8dbcbec40c5d1b9, entries=7, sequenceid=90, filesize=5.8 K 2024-12-02T14:22:07,258 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1742ms, sequenceid=90, compaction requested=false 2024-12-02T14:22:07,263 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-02T14:22:07,264 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T14:22:07,264 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T14:22:07,264 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733149325516Running coprocessor pre-close hooks at 1733149325516Disabling compacts and flushes for region at 1733149325516Disabling writes for close at 1733149325516Obtaining lock to block concurrent updates at 1733149325516Preparing flush snapshotting stores in 1588230740 at 1733149325516Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733149325517 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733149325517Flushing 1588230740/info: creating writer at 1733149325517Flushing 1588230740/info: appending metadata at 1733149325535 (+18 ms)Flushing 1588230740/info: closing flushed file at 1733149325535Flushing 1588230740/ns: creating writer at 1733149325949 (+414 ms)Flushing 1588230740/ns: appending metadata at 1733149325963 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733149325963Flushing 1588230740/rep_barrier: creating writer at 1733149326374 (+411 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733149326393 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733149326393Flushing 1588230740/table: creating writer at 1733149326806 (+413 ms)Flushing 1588230740/table: appending metadata at 1733149326824 (+18 ms)Flushing 1588230740/table: closing flushed file at 1733149326824Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bb514fd: reopening flushed file at 1733149327238 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@318d0cff: reopening flushed file at 1733149327244 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@280714ed: reopening flushed file at 1733149327249 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@675b5d07: reopening flushed file at 1733149327253 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1742ms, sequenceid=90, compaction requested=false at 1733149327258 (+5 ms)Writing region close event to WAL at 1733149327260 (+2 ms)Running coprocessor post-close hooks at 1733149327264 (+4 ms)Closed at 1733149327264 2024-12-02T14:22:07,264 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T14:22:07,319 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(976): stopping server a3a61c9ba14f,46793,1733149138543; all regions closed. 2024-12-02T14:22:07,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741834_1010 (size=25344) 2024-12-02T14:22:07,331 DEBUG [RS:0;a3a61c9ba14f:46793 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/oldWALs 2024-12-02T14:22:07,331 INFO [RS:0;a3a61c9ba14f:46793 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3a61c9ba14f%2C46793%2C1733149138543.meta:.meta(num 1733149141129) 2024-12-02T14:22:07,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741832_1008 (size=19989106) 2024-12-02T14:22:07,336 DEBUG [RS:0;a3a61c9ba14f:46793 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/oldWALs 2024-12-02T14:22:07,336 INFO [RS:0;a3a61c9ba14f:46793 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3a61c9ba14f%2C46793%2C1733149138543:(num 1733149140177) 2024-12-02T14:22:07,336 DEBUG [RS:0;a3a61c9ba14f:46793 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:22:07,336 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T14:22:07,336 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T14:22:07,337 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.ChoreService(370): Chore service for: regionserver/a3a61c9ba14f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T14:22:07,337 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T14:22:07,337 INFO [regionserver/a3a61c9ba14f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T14:22:07,338 INFO [RS:0;a3a61c9ba14f:46793 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46793 2024-12-02T14:22:07,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a3a61c9ba14f,46793,1733149138543 2024-12-02T14:22:07,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T14:22:07,341 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T14:22:07,342 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a3a61c9ba14f,46793,1733149138543] 2024-12-02T14:22:07,343 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a3a61c9ba14f,46793,1733149138543 already deleted, retry=false 2024-12-02T14:22:07,344 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a3a61c9ba14f,46793,1733149138543 expired; onlineServers=0 2024-12-02T14:22:07,344 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a3a61c9ba14f,35877,1733149137941' ***** 2024-12-02T14:22:07,344 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T14:22:07,344 INFO [M:0;a3a61c9ba14f:35877 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T14:22:07,344 INFO [M:0;a3a61c9ba14f:35877 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T14:22:07,344 DEBUG [M:0;a3a61c9ba14f:35877 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T14:22:07,344 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T14:22:07,344 DEBUG [M:0;a3a61c9ba14f:35877 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T14:22:07,344 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster-HFileCleaner.small.0-1733149139879 {}] cleaner.HFileCleaner(306): Exit Thread[master/a3a61c9ba14f:0:becomeActiveMaster-HFileCleaner.small.0-1733149139879,5,FailOnTimeoutGroup] 2024-12-02T14:22:07,344 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster-HFileCleaner.large.0-1733149139877 {}] cleaner.HFileCleaner(306): Exit Thread[master/a3a61c9ba14f:0:becomeActiveMaster-HFileCleaner.large.0-1733149139877,5,FailOnTimeoutGroup] 2024-12-02T14:22:07,344 INFO [M:0;a3a61c9ba14f:35877 {}] hbase.ChoreService(370): Chore service for: master/a3a61c9ba14f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T14:22:07,345 INFO [M:0;a3a61c9ba14f:35877 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T14:22:07,345 DEBUG [M:0;a3a61c9ba14f:35877 {}] master.HMaster(1795): Stopping service threads 2024-12-02T14:22:07,345 INFO [M:0;a3a61c9ba14f:35877 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T14:22:07,345 INFO [M:0;a3a61c9ba14f:35877 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T14:22:07,345 ERROR [M:0;a3a61c9ba14f:35877 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-02T14:22:07,346 INFO [M:0;a3a61c9ba14f:35877 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T14:22:07,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T14:22:07,346 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T14:22:07,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:22:07,346 DEBUG [M:0;a3a61c9ba14f:35877 {}] zookeeper.ZKUtil(347): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T14:22:07,346 WARN [M:0;a3a61c9ba14f:35877 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T14:22:07,347 INFO [M:0;a3a61c9ba14f:35877 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/.lastflushedseqids 2024-12-02T14:22:07,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742455_1631 (size=72) 2024-12-02T14:22:07,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T14:22:07,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46793-0x1009b4c753c0001, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T14:22:07,444 INFO [RS:0;a3a61c9ba14f:46793 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T14:22:07,445 INFO [RS:0;a3a61c9ba14f:46793 {}] regionserver.HRegionServer(1031): Exiting; stopping=a3a61c9ba14f,46793,1733149138543; zookeeper connection closed. 2024-12-02T14:22:07,446 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@59a9a8a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@59a9a8a 2024-12-02T14:22:07,447 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-02T14:22:07,758 INFO [M:0;a3a61c9ba14f:35877 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T14:22:07,758 INFO [M:0;a3a61c9ba14f:35877 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T14:22:07,759 DEBUG [M:0;a3a61c9ba14f:35877 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T14:22:07,759 INFO [M:0;a3a61c9ba14f:35877 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T14:22:07,759 DEBUG [M:0;a3a61c9ba14f:35877 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T14:22:07,759 DEBUG [M:0;a3a61c9ba14f:35877 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T14:22:07,759 DEBUG [M:0;a3a61c9ba14f:35877 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T14:22:07,759 INFO [M:0;a3a61c9ba14f:35877 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=746.69 KB heapSize=912.55 KB 2024-12-02T14:22:07,782 DEBUG [M:0;a3a61c9ba14f:35877 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/20c3165ef2a84671950cf873eb2a82f8 is 82, key is hbase:meta,,1/info:regioninfo/1733149141221/Put/seqid=0 2024-12-02T14:22:07,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742456_1632 (size=5672) 2024-12-02T14:22:08,187 INFO [M:0;a3a61c9ba14f:35877 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2043 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/20c3165ef2a84671950cf873eb2a82f8 2024-12-02T14:22:08,212 DEBUG [M:0;a3a61c9ba14f:35877 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d760210e27e1436c9040915ee08dfe17 is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1733149297515/Put/seqid=0 2024-12-02T14:22:08,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742457_1633 (size=44363) 2024-12-02T14:22:08,273 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T14:22:08,274 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T14:22:08,274 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-02T14:22:08,617 INFO [M:0;a3a61c9ba14f:35877 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746.13 KB at sequenceid=2043 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d760210e27e1436c9040915ee08dfe17 2024-12-02T14:22:08,625 INFO [M:0;a3a61c9ba14f:35877 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d760210e27e1436c9040915ee08dfe17 2024-12-02T14:22:08,641 DEBUG [M:0;a3a61c9ba14f:35877 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/763094c17d784c569a7716b66b26521e is 69, key is a3a61c9ba14f,46793,1733149138543/rs:state/1733149139924/Put/seqid=0 2024-12-02T14:22:08,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073742458_1634 (size=5156) 2024-12-02T14:22:09,046 INFO [M:0;a3a61c9ba14f:35877 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2043 (bloomFilter=true), to=hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/763094c17d784c569a7716b66b26521e 2024-12-02T14:22:09,055 DEBUG [M:0;a3a61c9ba14f:35877 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/20c3165ef2a84671950cf873eb2a82f8 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/20c3165ef2a84671950cf873eb2a82f8 2024-12-02T14:22:09,061 INFO [M:0;a3a61c9ba14f:35877 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/20c3165ef2a84671950cf873eb2a82f8, entries=8, sequenceid=2043, filesize=5.5 K 2024-12-02T14:22:09,062 DEBUG [M:0;a3a61c9ba14f:35877 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d760210e27e1436c9040915ee08dfe17 as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d760210e27e1436c9040915ee08dfe17 2024-12-02T14:22:09,067 INFO [M:0;a3a61c9ba14f:35877 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d760210e27e1436c9040915ee08dfe17 2024-12-02T14:22:09,067 INFO [M:0;a3a61c9ba14f:35877 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d760210e27e1436c9040915ee08dfe17, entries=164, sequenceid=2043, filesize=43.3 K 2024-12-02T14:22:09,068 DEBUG [M:0;a3a61c9ba14f:35877 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/763094c17d784c569a7716b66b26521e as hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/763094c17d784c569a7716b66b26521e 2024-12-02T14:22:09,071 INFO [M:0;a3a61c9ba14f:35877 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38859/user/jenkins/test-data/006f4e04-0fd4-cda8-3ecc-843c8a2e5919/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/763094c17d784c569a7716b66b26521e, entries=1, sequenceid=2043, filesize=5.0 K 2024-12-02T14:22:09,072 INFO [M:0;a3a61c9ba14f:35877 {}] regionserver.HRegion(3140): Finished flush of dataSize ~746.69 KB/764607, heapSize ~912.26 KB/934152, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1313ms, sequenceid=2043, compaction requested=false 2024-12-02T14:22:09,073 INFO [M:0;a3a61c9ba14f:35877 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T14:22:09,073 DEBUG [M:0;a3a61c9ba14f:35877 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733149327759Disabling compacts and flushes for region at 1733149327759Disabling writes for close at 1733149327759Obtaining lock to block concurrent updates at 1733149327759Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733149327759Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=764607, getHeapSize=934392, getOffHeapSize=0, getCellsCount=2232 at 1733149327760 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733149327762 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733149327762Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733149327782 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733149327782Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733149328197 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733149328212 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733149328212Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733149328626 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733149328641 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733149328641Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ff13954: reopening flushed file at 1733149329054 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26e4f24f: reopening flushed file at 1733149329061 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d046017: reopening flushed file at 1733149329067 (+6 ms)Finished flush of dataSize ~746.69 KB/764607, heapSize ~912.26 KB/934152, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1313ms, sequenceid=2043, compaction requested=false at 1733149329072 (+5 ms)Writing region close event to WAL at 1733149329073 (+1 ms)Closed at 1733149329073 2024-12-02T14:22:09,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41597 is added to blk_1073741830_1006 (size=898577) 2024-12-02T14:22:09,076 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T14:22:09,076 INFO [M:0;a3a61c9ba14f:35877 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T14:22:09,076 INFO [M:0;a3a61c9ba14f:35877 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35877 2024-12-02T14:22:09,076 INFO [M:0;a3a61c9ba14f:35877 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T14:22:09,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T14:22:09,178 INFO [M:0;a3a61c9ba14f:35877 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T14:22:09,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35877-0x1009b4c753c0000, quorum=127.0.0.1:51703, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T14:22:09,187 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d1d0948{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T14:22:09,190 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@49b49063{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T14:22:09,190 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T14:22:09,190 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@58f8c8ee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T14:22:09,190 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6aa6ac55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/hadoop.log.dir/,STOPPED} 2024-12-02T14:22:09,192 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T14:22:09,193 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T14:22:09,193 WARN [BP-2028019360-172.17.0.2-1733149135234 heartbeating to localhost/127.0.0.1:38859 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T14:22:09,193 WARN [BP-2028019360-172.17.0.2-1733149135234 heartbeating to localhost/127.0.0.1:38859 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2028019360-172.17.0.2-1733149135234 (Datanode Uuid c66cc415-bb0e-464a-9540-4f25224042e8) service to localhost/127.0.0.1:38859 2024-12-02T14:22:09,195 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/cluster_c8d22f2e-58bf-6f09-e5bc-40f0041042bb/data/data1/current/BP-2028019360-172.17.0.2-1733149135234 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T14:22:09,195 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/cluster_c8d22f2e-58bf-6f09-e5bc-40f0041042bb/data/data2/current/BP-2028019360-172.17.0.2-1733149135234 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T14:22:09,195 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T14:22:09,201 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T14:22:09,202 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T14:22:09,202 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T14:22:09,202 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T14:22:09,202 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/afd4a9b9-bb81-191b-9308-9ba09dfe5338/hadoop.log.dir/,STOPPED} 2024-12-02T14:22:09,217 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T14:22:09,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down